Zorba - NoSQL Query Processing

Content of full-text module

xquery version "3.0";

(:
 : Copyright 2006-2011 The FLWOR Foundation.
 :
 : Licensed under the Apache License, Version 2.0 (the "License");
 : you may not use this file except in compliance with the License.
 : You may obtain a copy of the License at
 :
 : http://www.apache.org/licenses/LICENSE-2.0
 :
 : Unless required by applicable law or agreed to in writing, software
 : distributed under the License is distributed on an "AS IS" BASIS,
 : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 : See the License for the specific language governing permissions and
 : limitations under the License.
 :)

(:===========================================================================:)

(:~
 : This module provides an XQuery API to full-text functions.
 : For general information about this implementation of the
 : &lt;a href="http://www.w3.org/TR/xpath-full-text-10/"&gt;XQuery and XPath Full Text 1.0 specification&lt;/a&gt;
 : as well as instructions for building an installing a thesaurus,
 : see the &lt;a href="../zorba/ft_thesaurus.html"&gt;Full Text Thesaurus documentation&lt;/a&gt;.
 : &lt;h2&gt;Notes on languages&lt;/h2&gt;
 : To refer to particular human languages,
 : uses either the
 : &lt;a href="http://en.wikipedia.org/wiki/ISO_639-1"&gt;ISO 639-1&lt;/a&gt;
 : or
 : &lt;a href="http://en.wikipedia.org/wiki/ISO_639-2"&gt;ISO 639-2&lt;/a&gt;
 : languages codes.
 : Note that only a subset of the
 : &lt;a href="http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes"&gt;complete list of language codes&lt;/a&gt;
 : are supported and not every function supports the same subset.
 : &lt;p/&gt;
 : Most functions in this module take a language as a parameter
 : using the
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;
 : XML schema data type.
 : &lt;h2&gt;Notes on stemming&lt;/h2&gt;
 : The &lt;code&gt;stem()&lt;/code&gt; functions return the
 : &lt;a href="http://en.wikipedia.org/wiki/Word_stem"&gt;stem&lt;/a&gt;
 : of a word.
 : The stem of a word itself, however, is not guaranteed to be a word.
 : It is best to consider a stem as an opaque byte sequence.
 : All that is guaranteed about a stem is that,
 : for a given word,
 : the stem of that word will always be the same byte sequence.
 : Hence,
 : you should never compare the result of one of the &lt;code&gt;stem()&lt;/code&gt;
 : functions against a non-stemmed string,
 : for example:
 : &lt;pre&gt;
 :  if ( ft:stem( "apples" ) eq "apple" )             ** WRONG **
 : &lt;/pre&gt;
 : Instead do:
 : &lt;pre&gt;
 :  if ( ft:stem( "apples" ) eq ft:stem( "apple" ) )  ** CORRECT **
 : &lt;/pre&gt;
 : &lt;h2&gt;Notes on the thesaurus&lt;/h2&gt;
 : The &lt;code&gt;thesaurus-lookup()&lt;/code&gt; functions have "levels"
 : and "relationship" parameters.
 : The values for these are implementation-defined.
 : The default implementation uses the
 : &lt;a href="http://wordnet.princeton.edu/"&gt;WordNet lexical database&lt;/a&gt;,
 : version 3.0.
 : &lt;p/&gt;
 : In WordNet,
 : the number of "levels" that two phrases are apart
 : are how many hierarchical meanings apart they are.
 : For example,
 : "canary" is 5 levels away from "vertebrate"
 : (carary &amp;gt; finch &amp;gt; oscine &amp;gt; passerine &amp;gt; bird &amp;gt; vertebrate).
 : &lt;p/&gt;
 : When using the WordNet implementation,
 : all of the relationships (and their abbreviations)
 : specified by
 : &lt;a href="http://www.iso.org/iso/iso_catalogue/catalogue_tc/catalogue_detail.htm?csnumber=7776"&gt;ISO 2788&lt;/a&gt;
 : and
 : &lt;a href="http://www.niso.org/kst/reports/standards?step=2&amp;amp;gid=&amp;amp;project_key=7cc9b583cb5a62e8c15d3099e0bb46bbae9cf38a"&gt;ANSI/NISO Z39.19-2005&lt;/a&gt;
 : with the exceptions of "HN" (history note)
 : and "X SN" (see scope note for) are supported.
 : These relationships are:
 :  &lt;table&gt;
 :    &lt;tr&gt;
 :      &lt;th&gt;Rel.&lt;/th&gt;
 :      &lt;th&gt;Meaning&lt;/th&gt;
 :      &lt;th&gt;WordNet Rel.&lt;/th&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;BT&lt;/td&gt;
 :      &lt;td&gt;broader term&lt;/td&gt;
 :      &lt;td&gt;hypernym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;BTG&lt;/td&gt;
 :      &lt;td&gt;broader term generic&lt;/td&gt;
 :      &lt;td&gt;hypernym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;BTI&lt;/td&gt;
 :      &lt;td&gt;broader term instance&lt;/td&gt;
 :      &lt;td&gt;instance hypernym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;BTP&lt;/td&gt;
 :      &lt;td&gt;broader term partitive&lt;/td&gt;
 :      &lt;td&gt;part meronym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;NT&lt;/td&gt;
 :      &lt;td&gt;narrower term&lt;/td&gt;
 :      &lt;td&gt;hyponym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;NTG&lt;/td&gt;
 :      &lt;td&gt;narrower term generic&lt;/td&gt;
 :      &lt;td&gt;hyponym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;NTI&lt;/td&gt;
 :      &lt;td&gt;narrower term instance&lt;/td&gt;
 :      &lt;td&gt;instance hyponym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;NTP&lt;/td&gt;
 :      &lt;td&gt;narrower term partitive&lt;/td&gt;
 :      &lt;td&gt;part holonym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;RT&lt;/td&gt;
 :      &lt;td&gt;related term&lt;/td&gt;
 :      &lt;td&gt;also see&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;SN&lt;/td&gt;
 :      &lt;td&gt;scope note&lt;/td&gt;
 :      &lt;td&gt;n/a&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;TT&lt;/td&gt;
 :      &lt;td&gt;top term&lt;/td&gt;
 :      &lt;td&gt;hypernym&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;UF&lt;/td&gt;
 :      &lt;td&gt;non-preferred term&lt;/td&gt;
 :      &lt;td&gt;n/a&lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;USE&lt;/td&gt;
 :      &lt;td&gt;preferred term&lt;/td&gt;
 :      &lt;td&gt;n/a&lt;/td&gt;
 :    &lt;/tr&gt;
 :  &lt;/table&gt;
 : Note that you can specify relationships
 : either by their abbreviation
 : or their meaning.
 : Relationships are case-insensitive.
 :
 : In addition to the
 : &lt;a href="http://www.iso.org/iso/iso_catalogue/catalogue_tc/catalogue_detail.htm?csnumber=7776"&gt;ISO 2788&lt;/a&gt;
 : and
 : &lt;a href="http://www.niso.org/kst/reports/standards?step=2&amp;amp;gid=&amp;amp;project_key=7cc9b583cb5a62e8c15d3099e0bb46bbae9cf38a"&gt;ANSI/NISO Z39.19-2005&lt;/a&gt;
 : relationships,
 : All of the relationships offered by WordNet are also supported.
 : These relationships are:
 :  &lt;table class="ft_rels"&gt;
 :    &lt;tr&gt;
 :      &lt;th&gt;Relationship&lt;/th&gt;
 :      &lt;th&gt;Meaning&lt;/th&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;also see&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that is related to another,
 :        e.g., for "varnished" (furniture)
 :        one should &lt;em&gt;also see&lt;/em&gt; "finished."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;antonym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word opposite in meaning to another,
 :        e.g., "light" is an &lt;em&gt;antonym&lt;/em&gt; for "heavy."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;attribute&lt;/td&gt;
 :      &lt;td&gt;
 :        A noun for which adjectives express values,
 :        e.g., "weight" is an &lt;em&gt;attribute&lt;/em&gt;
 :        for which the adjectives "light" and "heavy"
 :        express values.
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;cause&lt;/td&gt;
 :      &lt;td&gt;
 :        A verb that causes another,
 :        e.g., "show" is a &lt;em&gt;cause&lt;/em&gt; of "see."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;derivationally related form&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that is derived from a root word,
 :        e.g., "metric" is a &lt;em&gt;derivationally related form&lt;/em&gt; of "meter."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;derived from adjective&lt;/td&gt;
 :      &lt;td&gt;
 :        An adverb that is derived from an adjective,
 :        e.g., "correctly" is &lt;em&gt;derived from the adjective&lt;/em&gt; "correct."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;entailment&lt;/td&gt;
 :      &lt;td&gt;
 :        A verb that presupposes another,
 :        e.g., "snoring" &lt;em&gt;entails&lt;/em&gt; "sleeping."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;hypernym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word with a broad meaning that more specific words fall under,
 :        e.g., "meal" is a &lt;em&gt;hypernym&lt;/em&gt; of "breakfast."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;hyponym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word of more specific meaning than a general term applicable to it,
 :        e.g., "breakfast" is a &lt;em&gt;hyponym&lt;/em&gt; of "meal."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;instance hypernym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that denotes a category of some specific instance,
 :        e.g., "author" is an &lt;em&gt;instance hypernym&lt;/em&gt; of "Asimov."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;instance hyponym&lt;/td&gt;
 :      &lt;td&gt;
 :        A term that donotes a specific instance of some general category,
 :        e.g., "Asimov" is an &lt;em&gt;instance hyponym&lt;/em&gt; of "author."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;member holonym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that denotes a collection of individuals,
 :        e.g., "faculty" is a &lt;em&gt;member holonym&lt;/em&gt; of "professor."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;member meronym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that denotes a member of a larger group,
 :        e.g., a "person" is a &lt;em&gt;member meronym&lt;/em&gt; of a "crowd."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;part holonym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that denotes a larger whole comprised of some part,
 :        e.g., "car" is a &lt;em&gt;part holonym&lt;/em&gt; of "engine."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;part meronym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that denotes a part of a larger whole,
 :        e.g., an "engine" is &lt;em&gt;part meronym&lt;/em&gt; of a "car."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;participle of verb&lt;/td&gt;
 :      &lt;td&gt;
 :        An adjective that is the participle of some verb,
 :        e.g., "breaking" is the &lt;em&gt;participle of the verb&lt;/em&gt; "break."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td&gt;pertainym&lt;/td&gt;
 :      &lt;td&gt;
 :        An adjective that classifies its noun,
 :        e.g., "musical" is a &lt;em&gt;pertainym&lt;/em&gt; in "musical instrument."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;similar to&lt;/td&gt;
 :      &lt;td&gt;
 :        Similar, though not necessarily interchangeable, adjectives.
 :        For example, "shiny" is &lt;em&gt;similar to&lt;/em&gt; "bright",
 :        but they have subtle differences.
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;substance holonym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that denotes a larger whole containing some constituent
 :        substance, e.g., "bread" is a &lt;em&gt;substance holonym&lt;/em&gt; of "flour."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;substance meronym&lt;/td&gt;
 :      &lt;td&gt;
 :        A word that denotes a constituant substance of some larger whole,
 :        e.g., "flour" is a &lt;em&gt;substance meronym&lt;/em&gt; of "bread."
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :    &lt;tr&gt;
 :      &lt;td nowrap="nowrap"&gt;verb group&lt;/td&gt;
 :      &lt;td&gt;
 :        A verb that is a member of a group of similar verbs,
 :        e.g., "live" is in the &lt;em&gt;verb group&lt;/em&gt;
 :        of "dwell", "live", "inhabit", etc.
 :      &lt;/td&gt;
 :    &lt;/tr&gt;
 :  &lt;/table&gt;
 : &lt;h2&gt;Notes on tokenization&lt;/h2&gt;
 : For general information about the implementation of tokenization,
 : including what constitutes a token,
 : see the &lt;a href="../zorba/ft_tokenizer.html"&gt;Full Text Tokenizer&lt;/a&gt; documentation.
 : @author Paul J. Lucas
 : @project Zorba/Full-Text/Full-Text
 :)

module namespace ft = "http://zorba.io/modules/full-text";

declare namespace err = "http://www.w3.org/2005/xqt-errors";
declare namespace zerr = "http://zorba.io/errors";

declare namespace ver = "http://zorba.io/options/versioning";
declare option ver:module-version "1.0";

(:===========================================================================:)

(:~
 : Predeclared constant for the Danish
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-DA as xs:language := xs:language("da");

(:~
 : Predeclared constant for the German
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-DE as xs:language := xs:language("de");

(:~
 : Predeclared constant for the English
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-EN as xs:language := xs:language("en");

(:~
 : Predeclared constant for the Spanish
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-ES as xs:language := xs:language("es");

(:~
 : Predeclared constant for the Finnish
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-FI as xs:language := xs:language("fi");

(:~
 : Predeclared constant for the French
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-FR as xs:language := xs:language("fr");

(:~
 : Predeclared constant for the Hungarian
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-HU as xs:language := xs:language("hu");

(:~
 : Predeclared constant for the Italian
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-IT as xs:language := xs:language("it");

(:~
 : Predeclared constant for the Dutch
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-NL as xs:language := xs:language("nl");

(:~
 : Predeclared constant for the Norwegian
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-NO as xs:language := xs:language("no");

(:~
 : Predeclared constant for the Portuguese
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-PT as xs:language := xs:language("pt");

(:~
 : Predeclared constant for the Romanian
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-RO as xs:language := xs:language("ro");

(:~
 : Predeclared constant for the Russian
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-RU as xs:language := xs:language("ru");

(:~
 : Predeclared constant for the Swedish
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-SV as xs:language := xs:language("sv");

(:~
 : Predeclared constant for the Turkish
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;&lt;code&gt;xs:language&lt;/code&gt;&lt;/a&gt;.
 :)
declare variable $ft:LANG-TR as xs:language := xs:language("tr");

(:===========================================================================:)

(:~
 : Gets the current compare options.
 :
 : @return said compare options.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-current-compare-options-1.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-current-compare-options-2.xq
 :)
declare function ft:current-compare-options()
  as object() external;

(:~
 : Gets the current
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;:
 : either the language specified by the
 : &lt;code&gt;&lt;a href="http://www.w3.org/TR/xpath-full-text-10/#doc-xquery10-FTOptionDecl"&gt;declare ft-option using&lt;/a&gt;
 : &lt;a href="http://www.w3.org/TR/xpath-full-text-10/#ftlanguageoption"&gt;language&lt;/a&gt;&lt;/code&gt;
 : statement (if any)
 : or the one returned by &lt;code&gt;ft:host-lang()&lt;/code&gt; (if none).
 :
 : @return said language.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-current-lang-true-1.xq
 :)
declare function ft:current-lang()
  as xs:language external;

(:~
 : Gets the host's current
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;.
 : The "host" is the computer on which the software is running.
 : The host's current language is obtained as follows:
 :  &lt;ul&gt;
 :    &lt;li&gt;
 :      For *nix systems:
 :      &lt;ol&gt;
 :        &lt;li&gt;
 :          If &lt;a ref="http://www.cplusplus.com/reference/clibrary/clocale/setlocale/"&gt;&lt;code&gt;setlocale&lt;/code&gt;(3)&lt;/a&gt; returns non-null,
 :          the language corresponding to that locale is used.
 :        &lt;/li&gt;
 :        &lt;li&gt;
 :          Else, if the &lt;code&gt;LANG&lt;/code&gt; environment variable is set,
 :          that language is ued.
 :        &lt;/li&gt;
 :        &lt;li&gt;
 :          Otherwise, there is no default language.
 :        &lt;/li&gt;
 :      &lt;/ol&gt;
 :    &lt;/li&gt;
 :    &lt;li&gt;
 :      For Windows systems,
 :      the language corresponding to the locale returned by the
 :      &lt;a href="http://msdn.microsoft.com/en-us/library/windows/desktop/dd318101(v=vs.85).aspx"&gt;&lt;code&gt;GetLocaleInfo()&lt;/code&gt;&lt;/a&gt;
 :      function is used.
 :    &lt;/li&gt;
 :  &lt;/ul&gt;
 :
 : @return said language.
 :)
declare function ft:host-lang()
  as xs:language external;

(:~
 : Checks whether the given
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is supported for stemming.
 :
 : @param $lang The language to check.
 : @return &lt;code&gt;true&lt;/code&gt; only if the language is supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stem-lang-es-supported-true.xq
 :)
declare function ft:is-stem-lang-supported( $lang as xs:language )
  as xs:boolean external;

(:~
 : Checks whether the given word is a stop-word.
 :
 : @param $word The word to check.
 : @param $lang The
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : of &lt;code&gt;$word&lt;/code&gt;.
 : @return &lt;code&gt;true&lt;/code&gt; only if &lt;code&gt;$word&lt;/code&gt; is a stop-word.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stop-word-true-1.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stop-word-true-3.xq
 :)
declare function ft:is-stop-word( $word as xs:string, $lang as xs:language )
  as xs:boolean external;

(:~
 : Checks whether the given word is a stop-word.
 :
 : @param $word The word to check.
 : The word's &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is assumed to be the one returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 : @return &lt;code&gt;true&lt;/code&gt; only if &lt;code&gt;$word&lt;/code&gt; is a stop-word.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stop-word-true-2.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stop-word-true-4.xq
 :)
declare function ft:is-stop-word( $word as xs:string )
  as xs:boolean external;

(:~
 : Checks whether the given
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is supported for stop words.
 :
 : @param $lang The language to check.
 : @return &lt;code&gt;true&lt;/code&gt; only if the language is supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stop-word-lang-en-supported-true.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stop-word-lang-supported-false-1.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-stop-word-lang-supported-false-2.xq
 :)
declare function ft:is-stop-word-lang-supported( $lang as xs:language )
  as xs:boolean external;

(:~
 : Checks whether the given
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is supported for look-up using the default thesaurus.
 :
 : @param $lang The language to check.
 : @return &lt;code&gt;true&lt;/code&gt; only if the language is supported.
 :)
declare function ft:is-thesaurus-lang-supported( $lang as xs:language )
  as xs:boolean external;

(:~
 : Checks whether the given
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is supported for look-up using the thesaurus specified by the given URI.
 :
 : @param $uri The URI specifying the thesaurus to use.
 : @param $lang The language to check.
 : @return &lt;code&gt;true&lt;/code&gt; only if the language is supported.
 : @error err:FTST0018 if &lt;code&gt;$uri&lt;/code&gt; refers to a thesaurus
 : that is not found in the statically known thesauri.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-is-thesaurus-lang-supported-true-1.xq
 :)
declare function ft:is-thesaurus-lang-supported( $uri as xs:string,
                                                 $lang as xs:language )
  as xs:boolean external;

(:~
 : Checks whether the given
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is supported for tokenization.
 :
 : @param $lang The language to check.
 : @return &lt;code&gt;true&lt;/code&gt; only if the language is supported.
 :)
declare function ft:is-tokenizer-lang-supported( $lang as xs:language )
  as xs:boolean external;

(:~
 : Stems the given word.
 :
 : @param $word The word to stem.
 : @param $lang The
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : of &lt;code&gt;$word&lt;/code&gt;.
 : @return the stem of &lt;code&gt;$word&lt;/code&gt;.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-stem-1.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-stem-2.xq
 :)
declare function ft:stem( $word as xs:string, $lang as xs:language )
  as xs:string external;

(:~
 : Stems the given word.
 :
 : @param $word The word to stem.
 : The word's &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is assumed to be the one returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 : @return the stem of &lt;code&gt;$word&lt;/code&gt;.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-stem-3.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-stem-4.xq
 :)
declare function ft:stem( $word as xs:string )
  as xs:string external;

(:~
 : Strips all diacritical marks from all characters.
 :
 : @param $string The string to strip diacritical marks from.
 : @return &lt;code&gt;$string&lt;/code&gt; with diacritical marks stripped.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-strip-diacritics-1.xq
 :)
declare function ft:strip-diacritics( $string as xs:string )
  as xs:string external;

(:~
 : Looks-up the given phrase in the default thesaurus.
 :
 : @param $phrase The phrase to look up.
 : The phrase's
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is assumed to be the one returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 : @return the related phrases if &lt;code&gt;$phrase&lt;/code&gt; is found in the
 : thesaurus or the empty sequence if not.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is not supported.
 : @error zerr:ZXQP8401 if the thesaurus data file's version is not supported
 : by the currently running version of the software.
 : @error zerr:ZXQP8402 if the thesaurus data file's endianness does not match
 : that of the CPU on which the software is currently running.
 : @error zerr:ZXQP8403 if there was an error reading the thesaurus data.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-thesaurus-lookup-1.xq
 :)
declare function ft:thesaurus-lookup( $phrase as xs:string )
  as xs:string* external;

(:~
 : Looks-up the given phrase in the thesaurus specified by the given URI.
 :
 : @param $uri The URI specifying the thesaurus to use.
 : @param $phrase The phrase to look up.
 : @param $lang The
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : of &lt;code&gt;$phrase&lt;/code&gt;.
 : @return the related phrases if &lt;code&gt;$phrase&lt;/code&gt; is found in the
 : thesaurus or the empty sequence if not.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @error err:FTST0018 if &lt;code&gt;$uri&lt;/code&gt; refers to a thesaurus
 : that is not found in the statically known thesauri.
 : @error zerr:ZOSE0001 if the thesaurus data file could not be found.
 : @error zerr:ZOSE0002 if the thesaurus data file is not a plain file.
 : @error zerr:ZXQP8401 if the thesaurus data file's version is not supported
 : by the currently running version of the software.
 : @error zerr:ZXQP8402 if the thesaurus data file's endianness does not match
 : that of the CPU on which the software is currently running.
 : @error zerr:ZXQP8403 if there was an error reading the thesaurus data file.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-thesaurus-lookup-2.xq
 :)
declare function ft:thesaurus-lookup( $uri as xs:string, $phrase as xs:string,
                                      $lang as xs:language )
  as xs:string* external;

(:~
 : Looks-up the given phrase in a thesaurus.
 :
 : @param $uri The URI specifying the thesaurus to use.
 : @param $phrase The phrase to look up.
 : The phrase's
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is assumed to be the one the one returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 : @return the related phrases if &lt;code&gt;$phrase&lt;/code&gt; is found in the
 : thesaurus or the empty sequence if not.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is unsupported.
 : @error err:FTST0018 if &lt;code&gt;$uri&lt;/code&gt; refers to a thesaurus
 : that is not found in the statically known thesauri.
 : @error zerr:ZOSE0001 if the thesaurus data file could not be found.
 : @error zerr:ZOSE0002 if the thesaurus data file is not a plain file.
 : @error zerr:ZXQP8401 if the thesaurus data file's version is not supported
 : by the currently running version of the software.
 : @error zerr:ZXQP8402 if the thesaurus data file's endianness does not match
 : that of the CPU on which the software is currently running.
 : @error zerr:ZXQP8403 if there was an error reading the thesaurus data file.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-thesaurus-lookup-3.xq
 :)
declare function ft:thesaurus-lookup( $uri as xs:string, $phrase as xs:string )
  as xs:string* external;

(:~
 : Looks-up the given phrase in a thesaurus.
 :
 : @param $uri The URI specifying the thesaurus to use.
 : @param $phrase The phrase to look up.
 : @param $lang The
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : of &lt;code&gt;$phrase&lt;/code&gt;.
 : @param $relationship The relationship the results are to have to
 : &lt;code&gt;$phrase&lt;/code&gt;.
 : @return the related phrases if &lt;code&gt;$phrase&lt;/code&gt; is found in the
 : thesaurus or the empty sequence if not.
 : @error err:FTST0018 if &lt;code&gt;$uri&lt;/code&gt; refers to a thesaurus
 : that is not found in the statically known thesauri.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @error zerr:ZOSE0001 if the thesaurus data file could not be found.
 : @error zerr:ZOSE0002 if the thesaurus data file is not a plain file.
 : @error zerr:ZXQP8401 if the thesaurus data file's version is not supported
 : by the currently running version of the software.
 : @error zerr:ZXQP8402 if the thesaurus data file's endianness does not match
 : that of the CPU on which the software is currently running.
 : @error zerr:ZXQP8403 if there was an error reading the thesaurus data file.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-thesaurus-lookup-4.xq
 :)
declare function ft:thesaurus-lookup( $uri as xs:string, $phrase as xs:string,
                                      $lang as xs:language,
                                      $relationship as xs:string )
  as xs:string* external;

(:~
 : Looks-up the given phrase in a thesaurus.
 :
 : @param $uri The URI specifying the thesaurus to use.
 : @param $phrase The phrase to look up.
 : @param $lang The
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : of &lt;code&gt;$phrase&lt;/code&gt;.
 : @param $relationship The relationship the results are to have to
 : &lt;code&gt;$phrase&lt;/code&gt;.
 : @param $level-least The minimum number of levels within the thesaurus to be
 : traversed.
 : @param $level-most The maximum number of levels within the thesaurus to be
 : traversed.
 : @return the related phrases if &lt;code&gt;$phrase&lt;/code&gt; is found in the
 : thesaurus or the empty sequence if not.
 : @error err:FOCA0003 if either &lt;code&gt;$level-least&lt;/code&gt; or
 : &lt;code&gt;$level-most&lt;/code&gt; is either negative or too large.
 : @error err:FTST0018 if &lt;code&gt;$uri&lt;/code&gt; refers to a thesaurus
 : that is not found in the statically known thesauri.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @error zerr:ZOSE0001 if the thesaurus data file could not be found.
 : @error zerr:ZOSE0002 if the thesaurus data file is not a plain file.
 : @error zerr:ZXQP8401 if the thesaurus data file's version is not supported
 : by the currently running version of the software.
 : @error zerr:ZXQP8402 if the thesaurus data file's endianness does not match
 : that of the CPU on which the software is currently running.
 : @error zerr:ZXQP8403 if there was an error reading the thesaurus data file.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-thesaurus-lookup-5.xq
 :)
declare function ft:thesaurus-lookup( $uri as xs:string, $phrase as xs:string,
                                      $lang as xs:language,
                                      $relationship as xs:string,
                                      $level-least as xs:integer,
                                      $level-most as xs:integer )
  as xs:string* external;

(:~
 : Tokenizes the given node and all of its decendants.
 :
 : @param $node The node to tokenize.
 : @param $lang The default
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : of &lt;code&gt;$node&lt;/code&gt;.
 : @return a (possibly empty) sequence of tokens.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-node-1.xq
 :)
declare function ft:tokenize-node( $node as node(), $lang as xs:language )
  as object()* external;

(:~
 : Tokenizes the given node and all of its descendants.
 :
 : @param $node The node to tokenize.
 : The node's default
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is assumed to be the one returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 : @return a (possibly empty) sequence of tokens.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-node-2.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-node-3.xq
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-node-4.xq
 :)
declare function ft:tokenize-node( $node as node() )
  as object()* external;

(:~
 : Tokenizes the set of nodes comprising &lt;code&gt;$includes&lt;/code&gt; (and all of its
 : descendants) but excluding &lt;code&gt;$excludes&lt;/code&gt; (and all of its
 : descendants), if any.
 :
 : @param $includes The set of nodes (and its descendants) to include.
 : The default
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is assumed to be the one returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 : @param $excludes The set of nodes (and its descendants) to exclude.
 : @return a (possibly empty) sequence of tokens.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-nodes-1.xq
 :)
declare function ft:tokenize-nodes( $includes as node()+,
                                    $excludes as node()* )
  as object()* external;

(:~
 : Tokenizes the set of nodes comprising &lt;code&gt;$includes&lt;/code&gt; (and all of its
 : descendants) but excluding &lt;code&gt;$excludes&lt;/code&gt; (and all of its
 : descendants), if any.
 :
 : @param $includes The set of nodes (and its descendants) to include.
 : @param $excludes The set of nodes (and its descendants) to exclude.
 : @param $lang The default
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : for nodes.
 : @return a (possibly empty) sequence of tokens.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-nodes-1.xq
 :)
declare function ft:tokenize-nodes( $includes as node()+,
                                    $excludes as node()*,
                                    $lang as xs:language )
  as object()* external;

(:~
 : Tokenizes the given string.
 :
 : @param $string The string to tokenize.
 : @param $lang The
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : of &lt;code&gt;$string&lt;/code&gt;.
 : @return a (possibly empty) sequence of tokens.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-string-1.xq
 :)
declare function ft:tokenize-string( $string as xs:string,
                                     $lang as xs:language )
  as xs:string* external;

(:~
 : Tokenizes the given string.
 :
 : @param $string The string to tokenize.
 : The string's
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : is assumed to be the one returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 : @return a (possibly empty) sequence of tokens.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is not supported.
 : @example test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-string-2.xq
 :)
declare function ft:tokenize-string( $string as xs:string )
  as xs:string* external;

(:~
 : Gets properties of the tokenizer for the given
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;.
 :
 : @param $lang The language of the tokenizer to get the properties of.
 : @return said properties.
 : @error err:FTST0009 if &lt;code&gt;$lang&lt;/code&gt; is not supported.
 : tokenization specifically.
 :)
declare function ft:tokenizer-properties( $lang as xs:language )
  as object() external;

(:~
 : Gets properties of the tokenizer for the
 : &lt;a href="http://www.w3.org/TR/xmlschema-2/#language"&gt;language&lt;/a&gt;
 : returned by &lt;code&gt;ft:current-lang()&lt;/code&gt;.
 :
 : @return said properties.
 : @error err:FTST0009 if &lt;code&gt;ft:current-lang()&lt;/code&gt; is not supported.
 :)
declare function ft:tokenizer-properties()
  as object() external;

(:===========================================================================:)

(: vim:set et sw=2 ts=2: :)