Module:Hani-sortkey
- The following documentation is located at Module:Hani-sortkey/documentation. [edit] Categories were auto-generated by Module:module categorization. [edit]
- Useful links: subpage list • links • transclusions • testcases • sandbox (diff)
This module will sort text in the Han script. It is used to sort Ai-Cham, Southern Amami Ōshima, Central Bai, Panyi Bai, Southern Bai, Biao-Jiao Mien, Biyo, Eastern Min, Jin, Mandarin, Northern Pinghua, Chinese Pidgin English, Puxian Min, Macau Pidgin Portuguese, Southern Pinghua, Huizhou, Central Min, Dungan, Daur, E, Gan, Hakka, Yemaek, Hainanese, Xiang, Japanese, Hachijō, Kikai, Lama Bai, Middle Chinese, Leizhou Min, Literary Chinese, Jie, Rouran, Tuyuhun, Tuoba, Wuhuan, Xianbei, Middle Vietnamese, Caolan, Northern Min, Translingual, Miyako, Min Nan, Datian Min, Hokkien, Hailufeng Min, Longyan Min, Teochew, Zhenan Min, Sanxiang Min, Nùng, Old Chinese, Old Japanese, Okinoerabu, Old Uyghur, Bouyei, Baekje, Northern Amami Ōshima, Yaeyama, Okinawan, Shaojiang Min, Sui, Bailang, Tokunoshima, Alchuka, Bala, Kyakala, Tày, Vietnamese, Wu, Waxiang, Classical Tibetan, Middle Mongol, Buyeo, Kunigami, Yonaguni, Yoron, Cantonese, Zhuang, Zauzou, Chinese, Shaozhou Tuhua, Sichuanese, Taishanese, Goguryeo, Zakhring, Khitan, and Gaya.
The module should preferably not be called directly from templates or other modules.
To use it from a template, use {{sortkey}}
.
Within a module, use Module:languages#Language:makeSortKey.
For testcases, see Module:Hani-sortkey/testcases.
Functions
makeSortKey(text, lang, sc)
- Generates a sortkey for a given piece of
text
written in the script specified by the codesc
, and language specified by the codelang
. - When the sort fails, returns
nil
.
The demonstration functions that generated the content shown below are housed in Module:Hani-sortkey/templates. Modifications to the module can be tested in Module:Hani-sortkey/sandbox. Sortkeys for individual characters are retrieved from one of 178 data modules. Module:Hani-sortkey/data creates documentation for these modules.
Show sortkeys
- PS/2接口 (
PS2手08口00
) - gas爐 / gas炉 (
gas火16
) - γ粒子 (
γ米05子00
) - 命裡有時終須有,命裡無時莫強求 / 命里有时终须有,命里无时莫强求 (
口05衣07月02日06糸05頁03月02口05衣07火08日06艸07弓08水02
) - 得個……字 / 得个……字 (
彳08人08子03
) - 濕𣲷𣲷 / 湿𣲷𣲷 (
水14水05水05
) - 赛车 (
貝10車00
)
- 鿪 (
火13
)
Ideographic description sequences
- ⿰亻革 (
⿰人00革00
) - ⿰亻革家語 / ⿰亻革家语 (
⿰人00革00宀07言07
) - ⿺辶⿳穴⿲月⿱⿲幺言幺⿲長馬長刂心⿺辶⿳穴⿲月⿱⿲幺言幺⿲長馬長刂心麵 / ⿺辶⿳穴⿲月⿱⿲幺言幺⿲长马长刂心⿺辶⿳穴⿲月⿱⿲幺言幺⿲长马长刂心面 (
⿺辵00⿳穴00⿲月00⿱⿲幺00言00幺00⿲長00馬00長00刀00心00⿺辵00⿳穴00⿲月00⿱⿲幺00言00幺00⿲長00馬00長00刀00心00麥09
)
local export = {}
local m_str_utils = require("Module:string utilities")
local byte = string.byte
local codepoint = m_str_utils.codepoint
local concat = table.concat
local convert_iteration_marks = require("Module:Hani").convert_iteration_marks
local explode = m_str_utils.explode_utf8
local format = string.format
local gmatch = string.gmatch
local gsub = string.gsub
local insert = table.insert
local sub = string.sub
local u = m_str_utils.char
local ugsub = mw.ustring.gsub
local umatch = mw.ustring.match
local upper = m_str_utils.upper
local m_data = require("Module:Hani-sortkey/data/serialized")
local m_data_core = mw.loadData("Module:Hani-sortkey/data/core")
local cache = {}
--[[
Returns the index in the string where the ideographic description sequence
(IDS) ends, or the index of the end of the string. Iterates whenever
another ideographic description character (IDC) is found.
]]
local function findEndOfIDS(text, IDchar, i)
if not (text and IDchar and i) then
return nil
end
local j = i
local component = 1
-- Number of components expected after current IDC.
local components = m_data_core.ids[IDchar]
while component <= components do
j = j + 1
local char = text[j]
if not char then
break
elseif m_data_core.ids[char] then
j = findEndOfIDS(text, char, j)
end
component = component + 1
end
--[[
If the expected number of components has been found,
return the current index in the text.
]]
if component - components == 1 then
return j
else
return nil
end
end
local function unserialize(a, b)
return m_data_core.radicals[byte(a)] .. format("%02d", byte(b) - 10)
end
-- The data is stored in [[Module:Hani-sortkey/data]]. This data is not accessed directly (due to the large amount of memory this would consume), but is instead stored in a serialized form as [[Module:Hani-sortkey/data/serialized]]. If the data is changed, the new serialized data can be generated with [[Module:Hani-sortkey/data/serializer]].
function export.getData(char)
if type(char) == "string" then
char = codepoint(char)
elseif type(char) ~= "number" then
error("getData must operate on a single character or codepoint.")
end
local offset, s, f, lookup = 0
for i = 2, m_data_core.ranges.n, 2 do
s, f = m_data_core.ranges[i - 1], m_data_core.ranges[i]
if char > f then
offset = offset + f - s + 1
elseif char >= s and char <= f then
lookup = 2 * (offset + char - s + 1)
return (gsub(sub(m_data, lookup - 1, lookup), "(.)(.)", unserialize))
end
end
return u(char)
end
function export.makeSortKey(text, lang, sc)
local scripts = {
Hani = true,
Hans = true,
Hant = true,
Jpan = true,
Kore = true
}
if sc and not scripts[sc] then
return upper(text)
end
-- Convert any iteration marks into full characters, and remove any spaces. Also remove punctuation if the term contains non-punctuation (so that entries for punctuation characters can still be sorted properly).
text = ugsub(convert_iteration_marks(text), "%s+", "")
if not umatch(text, "^%p+$") then
text = ugsub(text, "%p+", "")
end
text = explode(text)
local sort, text_len, i = {}, #text, 0
while i < text_len do
i = i + 1
local char = text[i]
if m_data_core.preconvert[char] then
local j = 0
for c in gmatch(m_data_core.preconvert[char], ".[\128-\191]*") do
if j == 0 then
text[i] = c
else
insert(text, i + j, c)
end
j = j + 1
end
char = text[i]
text_len = #text
end
--[=[
If we encounter an ideographic description character (IDC),
find out if it begins a valid ideographic description sequence (IDS).
If the IDS is valid and a sortkey for it is listed in
[[Module:Hani-sortkey/data/unsupported]], then return
the sortkey, and move to the next character after the
IDS.
Otherwise, insert the IDC into the sortkey and move to the next
character after the IDC.
If the IDS is valid and no sortkey for it is found, track it.
]=]
if m_data_core.ids[char] then
local j = findEndOfIDS(text, char, i)
local IDS, data
if j then
IDS = concat(text, nil, i, j)
data = m_data_core.unsupported[IDS]
end
if not data then
if IDS then
require("Module:debug").track("Hani-sortkey/IDS-without-sortkey")
mw.log("ideographic description sequence without sortkey: '"
.. IDS .. "'")
else
require("Module:debug").track("Hani-sortkey/invalid-IDS")
mw.log("invalid ideographic description sequence at the beginning of '"
.. text[i] .. "'")
end
end
if IDS and data then
insert(sort, data)
i = j
else
insert(sort, char)
end
else
if not cache[char] then
cache[char] = export.getData(char)
end
insert(sort, cache[char])
end
end
return concat(sort)
end
return export
- Han script
- Sortkey-generating modules by script
- Han script modules
- Sortkey-generating modules
- Chinese Pidgin English modules
- Yonaguni modules
- Daur modules
- Southern Pinghua modules
- Shaojiang Min modules
- Jie modules
- Datian Min modules
- Central Bai modules
- Translingual modules
- Tuoba modules
- Ai-Cham modules
- Shaozhou Tuhua modules
- Zhuang modules
- Gaya modules
- Sichuanese modules
- Northern Pinghua modules
- Goguryeo modules
- Kikai modules
- Vietnamese modules
- Yoron modules
- Taishanese modules
- Zauzou modules
- Kyakala modules
- Sanxiang Min modules
- Nùng modules
- Literary Chinese modules
- Zakhring modules
- Gan modules
- Baekje modules
- Old Chinese modules
- Wuhuan modules
- Lama Bai modules
- Hakka modules
- Zhenan Min modules
- Puxian Min modules
- Yemaek modules
- Jin modules
- Macau Pidgin Portuguese modules
- Okinoerabu modules
- Chinese modules
- Bala modules
- Northern Min modules
- Japanese modules
- Kunigami modules
- Leizhou Min modules
- Dungan modules
- Hachijō modules
- Old Japanese modules
- Miyako modules
- Caolan modules
- E modules
- Wu modules
- Teochew modules
- Hainanese modules
- Min Nan modules
- Tuyuhun modules
- Xianbei modules
- Yaeyama modules
- Xiang modules
- Bouyei modules
- Tokunoshima modules
- Classical Tibetan modules
- Mandarin modules
- Central Min modules
- Longyan Min modules
- Middle Mongol modules
- Cantonese modules
- Eastern Min modules
- Waxiang modules
- Southern Amami Ōshima modules
- Northern Amami Ōshima modules
- Sui modules
- Tày modules
- Middle Vietnamese modules
- Alchuka modules
- Hokkien modules
- Middle Chinese modules
- Biao-Jiao Mien modules
- Okinawan modules
- Bailang modules
- Old Uyghur modules
- Khitan modules
- Hailufeng Min modules
- Biyo modules
- Buyeo modules
- Panyi Bai modules
- Huizhou modules
- Southern Bai modules
- Rouran modules