Module:User:IsomorphycSandbox/data tables
Appearance
- This module sandbox lacks a documentation subpage. Please create it.
- Useful links: root page • root page’s subpages • links • transclusions • testcases • sandbox
local export = {}
local base_location = "Module:data tables"
local metadata = mw.loadData(base_location.."/metadata")
local metadata = { ["shard_location_base"]={"Module:data tables/dataC"},["shards"]={100},["tables"] = {"grc_RWoodhouse_lemma_to_infinitives", "grc_RLBG_lemma_to_index", "grc_RWoodhouse_lemma_to_headwords", "freezer", "la_RMA_index_to_phrases", "grc_RCunliffe_lemma_to_index", "la_RMA_lemmas_no_collision_to_ix_phrase"}}
local shard_location_base = metadata["shard_location_base"][1] --"Module:data tables/data"
local shards = metadata["shards"][1]
local tables = metadata["tables"] --{"grc_RLBG_lemma_to_index", "grc_RWoodhouse_lemma_to_headwords", "grc_RWoodhouse_lemma_to_infinitives", "la_RMA_index_to_phrases", "grc_RCunliffe_lemma_to_index", "la_RMA_lemmas_no_collision_to_ix_phrase" }
function load_tables_unsharded()
function maybe_mw_load(x)
local success, val = pcall(function() return mw.loadData(x) end)
if success then return val else return nil end end
local unsharded, i = {}, 0
while #unsharded == i do
local val = maybe_mw_load(shard_location_base.."Unsharded"..tostring(i)) ---shard_location_base
if val ~= nil then
unsharded[#unsharded+1] = val end
i = i + 1
end
return unsharded
end
local unsharded = load_tables_unsharded()
function map(f, xs)
local ys = {}
for i,x in pairs(xs) do ys[i]=f(x) end
return ys
end
function map_string(f, s)
local ys = {}
for i = 1,#s do ys[i]=f(s:sub(i,i)) end
return ys
end
function fold(f, l, xs)
for i,x in pairs(xs) do l = f(l, x) end
return l
end
function hash(s)
return fold(function(x,y) return x + y end, 0, map_string(function(x) return x:byte() end, s))%shards
end
function export.list_tables()
return map(function (x) return x end, tables)
end
function export.load_table(n)
local t = {}
for i = 1,shards do
local curr = mw.loadData(shard_location_base..i)
if curr[n] ~= nil then
for k,v in pairs(curr[n]) do
t[k]=v end end end
for i,curr in pairs(unsharded) do
if curr[n] ~= nil then
for k,v in pairs(curr[n]) do
t[k]=v end end end
return t
end
-- eventually:
-- 1 : loop through all unsharded if multiple
-- 2 : index k in unsharded and shard separately, returning from unsharded preferentially
function export.index_table(t, k)
local shard = mw.loadData(shard_location_base..hash(k))
if unsharded[1] ~= nil and unsharded[1][t] ~= nil and unsharded[1][t][k] ~= nil then return unsharded[1][t][k] else if shard[t] ~= nil then return shard[t][k] else return nil end end
end
function export.index_table_all(t, ks)
baskets, result = {}, {}
for i,k in pairs(ks) do
h = hash(k)
if baskets[h] == nil then
baskets[h] = {k}
else
baskets[h][#(baskets[h])+1] = k
end
end
for h,b in pairs(baskets) do
local shard = mw.loadData(shard_location_base..h)
for i,k in pairs(b) do
if unsharded ~= nil and unsharded[1] ~= nil and unsharded[t] ~= nil then --here, loop over all unsharded if multiple
result[k] = unshared[t][k]
else
if shard[t][k] ~= nil then
result[k] = shard[t][k]
end
end
end
end
return result
end
function export.select(t,p)
return nil --Not implemented.
end
return export