Module:Cargo

--- -- --                             Module:Cargo -- -- Common tasks for the cargo extension are generalized into handy functions -- in this meta module ---

local getArgs = require('Module:Arguments').getArgs local m_util = require('Module:Util')

local cargo = mw.ext.cargo

-- The cfg table contains all localisable strings and configuration, to make it -- easier to port this module to another wiki. local cfg = mw.loadData('Module:Cargo/config')

local i18n = cfg.i18n

-- -- Exported functions --

local m_cargo = {}

-- -- Cargo function wrappers --

function m_cargo.declare(frame, args) return frame:callParserFunction('#cargo_declare:', args) end

function m_cargo.attach(frame, args) return frame:callParserFunction('#cargo_attach:', args) end

function m_cargo.store(frame, values, args) -- Calls the cargo_store parser function and ensures the values passed are casted properly --   -- Value handling: -- tables   - automatically concat -- booleans - automatically casted to 1 or 0 to ensure they're stored properly --   -- Arguments: -- frame        - frame object -- values       - table of field/value pairs to store -- args         - any additional arguments --  sep         - separator to use for concat --  store_empty - if specified, allow storing empty rows --  debug       - send the converted values to the lua debug log args = args or {} args.sep = args.sep or {} local i = 0 for k, v in pairs(values) do       i = i + 1 if type(v) == 'table' then if #v == 0 then i = i - 1 values[k] = nil else values[k] = table.concat(v, args.sep[k] or ',') end elseif type(v) == 'boolean' then if v == true then v = '1' elseif v == false then v = '0' end values[k] = v       end end -- i must be greater then 1 since we at least expect the _table argument to be set, even if no values are set if i > 1 or args.store_empty then if args.debug ~= nil then mw.logObject(values) end return frame:callParserFunction('#cargo_store:', values) end end

function m_cargo.query(tables, fields, query, args) -- Wrapper for mw.ext.cargo.query that helps to work around some bugs --   -- Current workarounds: -- field names will be "aliased" to themselves --   -- Takes 3 arguments: -- tables - array containing tables -- fields - array containing fields; these will automatically be renamed to the way they are specified to work around bugs when results are returned -- query  - array containing cargo sql clauses -- args --  args.keep_empty

-- Cargo bug workaround args = args or {} for i, field in ipairs(fields) do       -- already has some alternate name set, so do not do this. if string.find(field, '=') == nil then fields[i] = string.format('%s=%s', field, field) end end

query.limit = query.limit or cfg.limit*100 query.offset = query.offset or 0 local results = {} repeat local result = cargo.query(table.concat(tables, ','), table.concat(fields, ','), query) query.offset = query.offset + #result

for _,v in ipairs(result) do           results[#results + 1] = v        end until (#result < cfg.limit) or (#results >= query.limit)

if args.keep_empty == nil then for _, row in ipairs(results) do           for k, v in pairs(row) do                if v == "" then row[k] = nil end end end end return results end

-- -- Extended cargo functions --

function m_cargo.store_from_lua(args) -- Factory for function that stores data from lua data into a cargo table from a template call --   -- Arguments: -- module: Name of the module where the data is located, without the module prefix -- tables: Mapping of the table data --   -- Return: -- function that takes frame argument --   --    -- The function created takes the following arguments: -- REQURIED: --  tbl: table to store --  src: source wiki path after the module if it differs from the table name --  index_start: Starting index (default: 1) --  index_end: Ending index (default: data length, i.e. all data) args = args or {} if args.module == nil or args.tables == nil then error(i18n.errors.store_from_lua_missing_arguments) end

return function (frame) -- Get args tpl_args = getArgs(frame, {           parentFirst = true        }) frame = m_util.misc.get_frame(frame)

if args.tables[tpl_args.tbl] == nil then error(string.format(i18n.errors.store_from_lua_invalid_table, tostring(tpl_args.tbl))) end

-- mw.loadData has some problems... local data = require(string.format('%s:%s/%s', i18n.module, args.module, tpl_args.src or tpl_args.tbl))

tpl_args.index_start = math.max(tonumber(tpl_args.index_start) or 1, 1) tpl_args.index_end = math.min(tonumber(tpl_args.index_end) or #data, #data)

for i=tpl_args.index_start, tpl_args.index_end do           local row = data[i] if row == nil then break end -- get full table name row._table = args.tables[tpl_args.tbl].table m_cargo.store(frame, row) end

return string.format(i18n.tooltips.store_rows, tpl_args.index_start, tpl_args.index_end, tpl_args.index_end-tpl_args.index_start+1, tpl_args.tbl) end end

--[[test = {   tables = {    },    {        arg = {'argument1', 'argument2'},        header = 'Table header',        fields = {'mods.granted_skill'},        display = function(tpl_args, frame, tr, data)            tr                :tag('td')                    :wikitext(data['mods.granted_skill'])        end,        order = 1000,        sort_type = 'text',        options = '',    }, } =p.table_query{    tpl_args={        test=true,        q_where='mods.generation_type = 5 AND mods.domain = 1',        q_tables='spawn_weights',        q_join='mods._pageID=spawn_weights._pageID',    },    frame=nil,    main_table='mods',    --unique_row_fields={},    --empty_cell    data = {        tables = {            mod_stats = {join = 'mods._pageID=mod_stats._pageID'},        },        {            args = {'test'},            header = 'Id',            fields = {'mods.id'},            display = function(tpl_args, frame, tr, rows, rowinfo)                tr                    :tag('td')                        :wikitext(rows[1]['mods.id'])            end,            --order = 0,            sort_type = 'text',            --options = {},        },        {            args = {'test'},            header = 'Stats',            fields = {'mod_stats.id', 'mod_stats.min', 'mod_stats.max'},            display = function(tpl_args, frame, tr, rows, rowinfo)                local stats = {}                for _, row in ipairs(rows) do                    stats[#stats+1] = string.format('%s: %s to %s', row['mod_stats.id'], row['mod_stats.min'], row['mod_stats.max'])                end                tr                    :tag('td')                        :wikitext(table.concat(stats, ' '))            end,            --order = 0,            sort_type = 'text',            --options = {},        },    }, } ]]

function m_cargo.table_query(args) -- REQUIRED --  tpl_args --  frame --  main_table --  data --   tables --   [...]    --     args --    header --    fields --    display --    order --    sort_type --    options --     [...]    -- OPTIONAL --  row_unique_fields --  empty_cell --  table_css

-- TPL_ARGS: -- q_*** -- default -- before -- after -- *** - as defined in data local tpl_args = args.tpl_args local frame = m_util.misc.get_frame(args.frame) args.data.tables = args.data.tables or {} args.row_unique_fields = args.row_unique_fields or {string.format('%s._pageID', args.main_table)} args.empty_cell = args.empty_cell or ' '

local row_infos = {} for _, row_info in ipairs(args.data) do       local enabled = false if row_info.args == nil then enabled = true elseif type(row_info.args) == 'string' and m_util.cast.boolean(tpl_args[row_info.args]) then enabled = true elseif type(row_info.args) == 'table' then for _, argument in ipairs(row_info.args) do               if m_util.cast.boolean(tpl_args[argument]) then enabled = true break end end end

if enabled then row_info.options = row_info.options or {} row_infos[#row_infos+1] = row_info end end

-- sort the rows table.sort(row_infos, function (a, b)       return (a.order or 0) < (b.order or 0)    end)

-- Set tables local tables_assoc = { [args.main_table] = true, }   if tpl_args.q_tables then for _, tbl_name in ipairs(m_util.string.split(tpl_args.q_tables, ',%s*')) do           tables_assoc[tbl_name] = true end end

-- Set required fields local fields_assoc = { [string.format('%s._pageID', args.main_table)] = true, }   for _, rowinfo in ipairs(row_infos) do        if type(rowinfo.fields) == 'function' then rowinfo.fields = rowinfo.fields end for index, field in ipairs(rowinfo.fields) do           rowinfo.options[index] = rowinfo.options[index] or {} -- Support using functions such as CONCAT in fields: local f = string.match(               field, m_util.string.pattern.valid_var_name .. '%.'            ) if f ~= nil then tables_assoc[f] = true end fields_assoc[field] = true -- The results from the cargo query will use the aliased field: field = m_util.string.split(field, '%s*=%s*') rowinfo.fields[index] = field[2] or field[1] end end

for _, field in ipairs(args.row_unique_fields) do       fields_assoc[field] = true end

-- Parse query arguments local query = { }   for key, value in pairs(tpl_args) do        if string.sub(key, 0, 2) == 'q_' then query[string.sub(key, 3)] = value end end

if tpl_args.q_fields then local _extra_fields = m_util.string.split_outer(           tpl_args.q_fields,             ',%s*',             {'%(', '%)'}        ) for _, field in ipairs(_extra_fields) do           fields_assoc[field] = true end end

--   -- Query --   local tables = {args.main_table} local joins = {} for tbl_name, _ in pairs(tables_assoc) do       args.data.tables[tbl_name] = args.data.tables[tbl_name] or {} if args.data.tables[tbl_name].join then joins[#joins+1] = args.data.tables[tbl_name].join tables[#tables+1] = tbl_name elseif string.match(tpl_args.q_join or '', '.*' .. tbl_name .. '%..*') ~= nil then tables[#tables+1] = tbl_name elseif tbl_name ~= args.main_table then error(string.format(i18n.errors.no_join, tbl_name)) end end

local fields = {} for field, _ in pairs(fields_assoc) do       fields[#fields+1] = field end

if #joins > 0 then if query.join then query.join = query.join .. ',' .. table.concat(joins, ',') else query.join = table.concat(joins, ',') end end local results = {} local results_order = {} local cur_results = m_cargo.query(       tables,        fields,        query    ) for _, row in ipairs(cur_results) do       local unique_key = {} for _, field_name in ipairs(args.row_unique_fields) do           if row[field_name] == nil then error(string.format(i18n.errors.missing_unique_field_in_result_row, field_name, string.gsub(mw.dumpObject(row), '\n', ' '))) end unique_key[#unique_key+1] = row[field_name] end unique_key = table.concat(unique_key, '__') if results[unique_key] then table.insert(results[unique_key], row) else results[unique_key] = {row} results_order[#results_order+1] = unique_key end end

if #results_order == 0 then if tpl_args.default ~= nil then return tpl_args.default else return i18n.errors.no_results end end

--   -- Display --

-- Preformance optimization if tpl_args.q_fields then tpl_args._extra_fields = m_util.string.split_outer(           tpl_args.q_fields,             ',%s*',             {'%(', '%)'}        ) for index, field in ipairs(tpl_args._extra_fields) do           field = m_util.string.split(field, '%s*=%s*') -- field[2] will be nil if there is no alias tpl_args._extra_fields[index] = field[2] or field[1] end else tpl_args._extra_fields = {} end

local tbl = mw.html.create('table') tbl:attr('class', 'wikitable sortable ' .. (args.table_css or ''))

-- Header local tr = tbl:tag('tr') for _, row_info in ipairs(row_infos) do       tr            :tag('th') :attr('data-sort-type', row_info.sort_type or 'number') :wikitext(row_info.header) :done end

for _, field in ipairs(tpl_args._extra_fields) do       tr            :tag('th') :wikitext(field) end

-- Body

for _, unique_key in ipairs(results_order) do       local rows = results[unique_key] tr = tbl:tag('tr')

for _, rowinfo in ipairs(row_infos) do           local display_fields = {} for index, field in ipairs(rowinfo.fields) do               if rowinfo.options[index].optional ~= true then display_fields[field] = false for _, row in ipairs(rows) do                       if row[field] ~= nil then display_fields[field] = true break end end end end

local display = true for key, value in pairs(display_fields) do               if not value then display = false break end end

if display then rowinfo.display(tpl_args, frame, tr, rows, rowinfo) else tr:wikitext(args.empty_cell) end end

-- Add extra columns specified by tpl_args.q_fields: for _, field in ipairs(tpl_args._extra_fields) do           local extra_col = {} for _, row in ipairs(rows) do               if row[field] then extra_col[#extra_col+1] = row[field] end end if #extra_col > 0 then tr                   :tag('td') :wikitext(table.concat(extra_col, ' ')) else tr:wikitext(args.empty_cell) end end end

return (tpl_args.before or '') .. tostring(tbl) .. (tpl_args.after or '') end

function m_cargo.parse_field_arguments(args) -- Maps the arguments from a cargo argument table (i.e. the ones used in m_cargo.declare_factory) --   -- It will expect/handle the following fields: -- map.order              - REQUIRED - Array table for the order in which the arguments in map.fields will be parsed -- map.table              - REQUIRED - Table name (for storage) -- map.fields[id].field   - REQUIRED - Name of the field in cargo table -- map.fields[id].type    - REQUIRED - Type of the field in cargo table -- map.fields[id].func    - OPTIONAL - Function to handle the arguments. It will be passed tpl_args and frame. --                                     The function should return the parsed value. --   --                                      If no function is specified, default handling depending on the cargo field type will be used -- map.fields[id].default - OPTIONAL - Default value if the value is not set or returned as nil --                                     If default is a function, the function will be called with (tpl_args, frame) and expected to return a default value for the field. -- map.fields[id].name    - OPTIONAL - Name of the field in tpl_args if it differs from the id in map.fields. Used for i18n for example -- map.fields[id].required - OPTIONAL - Whether a value for the field is required or whether it can be left empty --                                     Note: With a default value the field will never be empty -- map.fields[id].skip    - OPTIONAL - Skip field if missing from order --   --    -- Expects argument table. -- REQUIRED: -- tpl_args  - arguments passed to template after preprecessing -- frame     - frame object -- table_map - table mapping object -- rtr       - if set return cargo props instead of storing them local tpl_args = args.tpl_args local frame = args.frame local map = args.table_map

local cargo_values = {_table = map.table}

-- for checking missing keys in order local available_fields = {} for key, field in pairs(map.fields) do       if field.skip == nil then available_fields[key] = true end end

-- main loop for _, key in ipairs(map.order) do       local field = map.fields[key] if field == nil then error(string.format(i18n.errors.missing_key_in_fields, key, map.table)) else available_fields[key] = nil end -- key in argument mapping local args_key if field.name then args_key = field.name else args_key = key end -- Retrieve value local value -- automatic handling only works if the field type is set if field.type ~= nil then value = tpl_args[args_key]

local cfield = m_cargo.parse_field{field=field.type} local handler if cfield.type == 'Integer' or cfield.type == 'Float' then handler = tonumber elseif cfield.type == 'Boolean' then handler = function (value) return m_util.cast.boolean(value, {cast_nil=false}) end end

if cfield.list and value ~= nil then -- ingore whitespace between separator and values value = m_util.string.split(value, cfield.list .. '%s*') if handler then for index, v in ipairs(value) do                       value[index] = handler(v) if value[index] == nil then error(string.format(i18n.errors.handler_returned_nil, map.table, args_key, v, field.type)) end end end elseif handler and value ~= nil then value = handler(value) if value == nil then error(string.format(i18n.errors.handler_returned_nil, map.table, args_key, tpl_args[args_key], field.type)) end end -- Don't need special handling: String, Text, Wikitext, Searchtext -- Consider: Page, Date, Datetime, Coordinates, File, URL, Email end if field.func ~= nil then value = field.func(tpl_args, frame, value) end -- Check defaults if value == nil and field.default ~= nil then if type(field.default) == 'function' then value = field.default(tpl_args, frame) elseif type(field.default) == 'table' then mw.logObject(string.format(i18n.errors.table_object_as_default, key, map.table)) value = mw.clone(field.default) else value = field.default end end -- Add value to arguments and cargo data if value ~= nil then -- key will be used here since the value will be used internally from here on in english tpl_args[key] = value if field.field ~= nil then cargo_values[field.field] = value end elseif field.required == true then error(string.format(i18n.errors.argument_required, args_key)) end end

-- check for missing keys and return error if any are missing local missing = {} for key, _ in pairs(available_fields) do       missing[#missing+1] = key end if #missing > 0 then error(string.format(i18n.errors.missing_key_in_order, map.table, table.concat(missing, '\n'))) end

-- finally store data in DB   if args.rtr ~= nil then return cargo_values else m_cargo.store(frame, cargo_values) end end

function m_cargo.declare_factory(args) -- Returns a function that can be called by templates to declare cargo tables --   -- args -- data: data table --  table: name of cargo table --  fields: associative table with: --   field: name of the field to declare --   type: type of the  field return function (frame) frame = m_util.misc.get_frame(frame)

local dcl_args = {} dcl_args._table = args.data.table for k, field_data in pairs(args.data.fields) do           if field_data.field then dcl_args[field_data.field] = field_data.type end end

return m_cargo.declare(frame, dcl_args) end end

function m_cargo.attach_factory(args) -- Returns a function that can be called by templates to attach cargo tables --   -- args -- data: data table --  table: name of cargo table --  fields: associative table with: --   field: name of the field to declare --   type: type of the  field return function (frame) frame = m_util.misc.get_frame(frame)

local attach_args = {} attach_args._table = args.data.table

return m_cargo.attach(frame, attach_args) end end

-- mw.logObject(m_cargo.map_results_to_id{results=mw.ext.cargo.query('mods,spawn_weights', 'mods._pageID, spawn_weights.tag', {where='mods.id="Strength1"', join='mods._pageID=spawn_weights._pageID'}), field='mods._pageID'}) function m_cargo.map_results_to_id(args) -- Maps the results passed to a table containing the specified field as key and a table of rows for the particular page as values. --   -- args -- results         : Table of results returned from mw.ext.cargo.query to map to the specified id field. -- field           : Name of the id field to map results to    --                    the field has to be in the fields list of the original query or it will cause errors. -- keep_id_field   : If set then don't delete _pageID. -- append_id_field : If set then append the id to the table sequentially as well which allows preserving --                   the id order they were found in. --   -- return -- table --  key         : The specified id field --  value       : Array containing the found rows (in the order that they were found) local out = {} for _, row in ipairs(args.results) do       local key = row[args.field] if out[key] then out[key][#out[key]+1] = row else out[key] = {row}

-- Append the ids sequentially, this allows preserving the order -- the ids were found: if args.append_id_field ~= nil then out[#out+1] = key end end

--Discard the pageID, don't need this any longer in most cases: if args.keep_id_field == nil then row[args.field] = nil end end

return out end

function m_cargo.array_query(args) -- Performs a long "OR" query from the given array and field validating that there is only exactly one match returned --   -- args: -- REQUIRED: --  tables    - array of tables (see m_cargo.query) --  fields    - array of fields (see m_cargo.query) --  id_array  - list of ids to query for --  id_field  - name of the id field, will be automatically added to fields -- OPTIONAL: --  query              - array containing cargo sql clauses [optional] (see m_cargo.query) --  ingore_missing     - skip the check for missing fields entirely --  warning_on_missing - issue warning instead of error if missing values --   -- RETURN: -- table - results as given by mw.ext.cargo.query -- msg - any error messages if it was used as warning args.query = args.query or {}

args.fields[#args.fields+1] = args.id_field

if #args.id_array == 0 then return {} end

-- remove blanks local id_array = {} for _, value in ipairs(args.id_array) do       if value ~= '' then id_array[#id_array+1] = value end end

-- for error returning local msg = {}

local where = string.format('%s IN ("%s")', args.id_field, table.concat(id_array, '","')) if args.query.where then args.query.where = string.format('(%s) AND (%s)', args.query.where, where) else args.query.where = where end

--   -- Prepare query --

local results = m_cargo.query(       args.tables,        args.fields,        args.query    )

--   -- Check missing results --   if #results ~= #id_array then --       -- Check for duplicates --       -- The usage of distinct should elimate duplicates here from cargo being bugged while still showing actual data duplicates. local dupes = m_cargo.query(           args.tables,            {                string.format('COUNT(DISTINCT %s._pageID)=count', args.tables[1]),                args.id_field,            },            {                join=args.query.join,                where=args.query.where,                groupBy=args.id_field,                having=string.format('COUNT(DISTINCT %s._pageID) > 1', args.tables[1]),            }        )

if #dupes > 0 then out = {} for _, row in ipairs(dupes) do               out[#out+1] = string.format('%s (%s pages found)', row[args.id_field], row['count']) end error(string.format(i18n.errors.duplicate_ids, args.id_field, table.concat(out, '\n'))) end

local dupes = m_cargo.query(           args.tables,            {                string.format('COUNT(%s)=count', args.id_field),                string.format('%s._pageName=page', args.tables[1]),            },            {                join=args.query.join,                where=args.query.where,                groupBy=string.format('%s._pageName', args.tables[1]),                having=string.format('COUNT(%s) > 1', args.id_field),            }        )

if #dupes > 0 then out = {} for _, row in ipairs(dupes) do               out[#out+1] = string.format('"%s" (%s entries found)', row['page'], row['count']) end error(string.format(i18n.errors.duplicate_ids, args.id_field, table.concat(out, '\n'))) end

if not args.ignore_missing then local missing = {} for _, id in ipairs(id_array) do               missing[id] = true end for _, row in ipairs(results) do               missing[row[args.id_field]] = nil end

local missing_ids = {} for k, _ in pairs(missing) do               missing_ids[#missing_ids+1] = k            end

msg[#msg+1] = string.format(i18n.errors.missing_ids, args.id_field, table.concat(missing_ids, '\n')) if args.warning_on_missing == nil then error(msg[#msg]) else mw.logObject(msg[#msg]) end end end

return results, msg end

function m_cargo.replace_holds(args) -- Replaces a holds query with a like or regexp equivalent. --   -- required args: -- string: string to replace --   -- optional args: -- mode     : Either "like" or "regex"; default "regex" --             like: Replaces the holds query with a LIKE equivalent --             regex: Replaces the holds query with a REGEXP equivalent -- field    : Field pattern to use. Can be used to restrict the hold replacement to specific fields. --            Default: all fields are matched. -- separator: Separator for field entries to use in the REGEXP mode. --            Default: , --   -- Returns the replaced query local args = args or {} -- if the field is not specified, replace any holds query args.field = args.field or '[%w_\.]+' if args.mode == 'like' or args.mode == nil then return string.gsub(           args.string,            string.format('(%s) HOLDS ([NOT ]*)([LIKE ]*)"([^"]+)"', args.field),            '%1__full %2LIKE "%%%4%%"'        )    elseif args.mode == 'regex' then        args.separator = args.separator or ','        return string.gsub(            args.string,            string.format('(%s) HOLDS ([NOT ]*)"([^"]+)"', args.field),            string.format('%%1__full %%2REGEXP "(%s|^)%%3(%s|$)"', args.separator, args.separator)        ) else error('Invalid mode specified. Acceptable values are like or regex.') end end

function m_cargo.parse_field(args) -- Parse a cargo field declaration and returns a table containing the results --   -- required args: -- field: field to parse --   -- Return -- type       - Type of the field -- list       - Separator of the list if it is a list type field -- parameters - any parameters to the field itself local field = args.field local results = {} local match

match = { string.match(field, 'List %(([^%(%)]+)%) of (.*)') } if #match > 0 then results.list = match[1] field = match[2] end

match = { string.match(field, '%s*(%a+)%s*%(([^%(%)]+)%)') } if #match > 0 then results.type = match[1] field = match[2] results.parameters = {} for _, param_string in ipairs(m_util.string.split(field, ';')) do           local index = { string.find(param_string, '=') } local key local value if #index > 0 then key = string.sub(param_string, 0, index[1]-1) value = m_util.string.strip(string.sub(param_string, index[1]+1)) else key = param_string value = true end results.parameters[m_util.string.strip(key)] = value end else -- the reminder must be the type since there is no extra declarations results.type = string.match(field, '%s*(%a+)%s*') end

return results end

return m_cargo