QScript Functions for Filters
Jump to navigation
Jump to search
This page contains functions for creating filters.
To make these functions available when writing a QScript or Rule see JavaScript Reference.
createFiltersForAllCategories(question)
Given a question, this function creates a new filter for each category in the corresponding question.
getMergedCategoriesFromPickOne(question)
Returns an array describing which categories in the pick one (- multi) question have been merged (excluding the NET). Each entry in the array has:
- name: The name of the merged category
- values: An array of the underlying values for the categories that have been merged
getMergedCategoriesFromPickAny(question)
Returns an array describing which categories in the pick any (- multi) question have been merged (excluding the NET). Each entry in the array has:
- name: The name of the merged category
- values: An array of the underlying values for the categories that have been merged
mergePickAnyExpression(variable_names)
Generate an JavaScript expression for merged categories in a Pick Any question.
mergePickAnyCompactExpression(variable_names, codes)
Generate an JavaScript expression for merged categories in a Pick Any - Compact question.
generateUniqueControlsName(name)
Helper function for createControlsAndFilter(). Ensures that all the created controls have unique names.
recursiveGetAllControlsNamesInGroup(group_item, objects_array)
Helper function for generateUniqueControlsName(). Recursively gets all controls in the group specified by group_item, e.g. project.report, and adds them to objects_array.
determineNonNETLabels(variable)
Uses Data Reduction to determine labels to show in the controls created by createControlsAndFilter(), doesn't match up with factor levels shown to R code exactly (most noticeably, hidden codes aren't returned by this function, but are present in the R factor representation of the variable. Used by reorderHiddenCodes.
readLevelsFromRFactor(v)
Given a variable, v, this function creates a temporary R Output to read levels of the R factor representation of the variable and returns them in an array.
Helper function for readLevelsFromRFactor(). Any hidden codes in a variable will still appear in the R variable, but unfortunately the codes/levels are in the wrong order, with the hidden codes always being the last levels of the R factor. This function attempts to correct this, reordering the labels using the underlying values. levels_fromR is an array of the the levels of the R factor representation of v, a variable and levels_excluding_hidden is the result of a call to determineNonNETLabels().
getDataFileFromDependants(deps)
Determines the DataFile given the dependants of an R Output.
createControlsAndFilter(control_type)
Main function for Combo Box (Drop-Down) Filters on an Output, Text Box Filters on an Output, and List Box Filters on an Output. Prompts the user for categorical variable sets to create a filter from, adds controls to the page to modify the created filter, creates the filter as an R variable, and applies it to any Plots, Tables, or R Outputs selected on the page by the user. Valid values for control_type are "Combobox", "Textbox", or "Listbox".
Source Code
includeWeb('QScript Questionnaire Functions');
includeWeb('QScript Value Attributes Functions');
includeWeb('QScript Utility Functions');
includeWeb('QScript Selection Functions');
includeWeb('QScript Functions to Generate Outputs');
includeWeb('QScript Data Reduction Functions');
includeWeb('QScript Functions for Combining Categories');
function createFiltersForAllCategories(question) {
var q_type = question.questionType;
var new_filter_question;
var new_name = preventDuplicateQuestionName(question.dataFile, question.name + " - Filters");
var merged_categories;
// Pick Any - Grid simply gets converted to a Pick Any because there is
// no way to treat merged categories.
if (q_type == "Pick Any - Grid") {
new_filter_question = question.duplicate(new_name);
new_filter_question.questionType = "Pick Any";
} else if (q_type == "Pick One - Multi") {
// Flatten and merge using existing methods
var merged_categories = getMergedCategoriesFromPickOne(question);
if (merged_categories == null || merged_categories.length == 0)
return null;
new_filter_question = pickOneMultiToPickAnyFlattenAndMergeByRows(question, merged_categories, true, false);
new_filter_question.name = preventDuplicateQuestionName(question.dataFile, new_filter_question.name.replace("(flattened)", "- Filters"));
new_filter_question.questionType = "Pick Any - Grid";
if (!new_filter_question.isValid)
{
new_filter_question.questionType = "Pick Any";
}
} else {
// For other question types we work out which categories are
// in the data reduction (excluding the main NET).
if (["Pick One", "Pick Any - Compact"].indexOf(q_type) > -1)
merged_categories = getMergedCategoriesFromPickOne(question);
else if (q_type == "Pick Any")
merged_categories = getMergedCategoriesFromPickAny(question);
else
throw "Not applicable for " + q_type + "questions.";
if (merged_categories == null)
return null;
// Create a new variable for each category in the data reduction
var variables = question.variables;
var data_file = question.dataFile;
var new_vars = [];
var num_vars = variables.length;
var last_var = variables[num_vars-1];
var new_name_prefix = "FILTER" + makeid() +"_";
merged_categories.forEach(function (obj, ind) {
var expression;
if(q_type == "Pick Any")
expression = mergePickAnyExpression(obj.variables);
else if (q_type == "Pick Any - Compact")
expression = mergePickAnyCompactExpression(variables.map(function (v) { return v.name; }), obj.values);
else if (q_type == "Pick One")
expression = mergePickOneExpression(variables[0].name, obj.values, true);
try {
var new_var = question.dataFile.newJavaScriptVariable(expression,
false,
new_name_prefix + ind + 1,
obj.name,
last_var,
{skipValidation: true, accelerate: true});
} catch (e) {
log("Could not create filter: " + e);
return false;
}
new_var.variableType = "Categorical";
last_var = new_var;
new_vars.push(new_var);
});
// Form the question
if (new_vars.length > 0) {
var new_q_name = preventDuplicateQuestionName(data_file, question.name + " - Filters");
var new_filter_question = data_file.setQuestion(new_q_name, "Pick Any", new_vars);
} else
return null;
}
// Set the properties of the new question
new_filter_question.isFilter = true;
setLabelForVariablesInQuestion(new_filter_question, 0, "Not Selected");
setLabelForVariablesInQuestion(new_filter_question, 1, "Selected");
return new_filter_question;
}
// Returns an array describing which categories in the pick one (- multi)
// question have been merged (excluding the NET). Each entry in the array has:
// - name: The name of the merged category
// - values: An array of the underlying values for the categories that have
// been merged
function getMergedCategoriesFromPickOne(q) {
var value_attributes = q.valueAttributes;
var non_missing_values = q.uniqueValues.filter(function (x) {
return !value_attributes.getIsMissingData(x);
}).sort();
// Get the set of values for each code in the data reduction
var merging_objects = getAllUnderlyingValues(q);
// Filter out the set of values corresoponding to the NET as
// we don't want to keep them.
merging_objects = merging_objects.filter(function (obj) {
return obj.array.sort().toString() != non_missing_values.toString();
});
merging_objects = merging_objects.map(function (obj) {
return { name: obj.label, values: obj.array };
});
return merging_objects;
}
// Returns an array describing the data reduction of a pick any question
// excluding the NET. Each entry in the array corresponds to a code from
// the data reduction, and has:
// - name: The name of the category.
// - variables: The names of the variables in the category.
function getMergedCategoriesFromPickAny(q) {
var data_reduction = q.dataReduction;
var net_labels = ["NET"];
if (fileFormatVersion() > 8.41)
net_labels = data_reduction.netRows.map(function (x) { return data_reduction.rowLabels[x]; });
var merging_objects = [];
data_reduction.rowLabels.forEach(function (label) {
if (net_labels.indexOf(label) == -1)
merging_objects.push({ name: label, variables: data_reduction.getUnderlyingVariables(label).map(function (v) { return v.name; }) });
});
return merging_objects;
}
// Generate an JavaScript expression for merged categories in a Pick Any question.
function mergePickAnyExpression(variable_names) {
var nan_bit = "isNaN(" + variable_names[0] + ")";
var main_bit = "(" + variable_names[0];
if (variable_names.length > 1) {
for (var j = 1; j < variable_names.length; j++) {
nan_bit += " || isNaN(" + variable_names[j] + ")";
main_bit += " || " + variable_names[j];
}
}
return nan_bit + " ? NaN : " + main_bit + ") ? 1 : 0;";
}
// Generate an JavaScript expression for merged categories in a Pick Any - Compact question.
function mergePickAnyCompactExpression(variable_names, codes) {
var missing_check = "(" + variable_names.map(function (name) { return "Q.IsMissing(" + name + ")"; } ).join(" && ") + ") ? NaN : (";
var term_array = [];
variable_names.forEach(function (name) {
codes.forEach(function (code) {
term_array.push("Q.Source(" + name + ") == " + code);
});
});
return missing_check + term_array.join(" || ") + ") ? 1 : 0;";
}
function generateUniqueControlsName(name) {
var controls = [];
recursiveGetAllControlsNamesInGroup(project.report, controls);
if (controls.indexOf(name) == -1)
return name;
var nonce = 1;
while (controls.indexOf(name + "." + nonce.toString()) != -1)
++nonce;
return name + "." + nonce.toString();
}
function recursiveGetAllControlsNamesInGroup(group_item, objects_array) {
var cur_sub_items = group_item.subItems;
for (var j = 0; j < cur_sub_items.length; j++) {
if (cur_sub_items[j].type == 'ReportGroup') {
recursiveGetAllControlsNamesInGroup(cur_sub_items[j], objects_array);
}
else if (cur_sub_items[j].type == 'Control') {
objects_array.push(cur_sub_items[j].referenceName);
objects_array.push(cur_sub_items[j].name);
}
}
}
// Uses QScript Data Reduction to determine labels
// to show in the List Boxes, doesn't match up with factor levels
// shown to R code exactly (most noticeably, hidden codes aren't
// returned by this function, but are present in the R factor
// Hence, this is only used as a fallback if simply reading
// the levels from the factor fails (it shouldn't)
function determineNonNETLabels(variable) {
var vals = getAllUnderlyingValues(variable.question);
var labels = [];
for (var i = 0; i < vals.length; i++) {
var val_arr = vals[i].array;
var isNET = false;
if (val_arr.length > 1) {
for (var j = 0; j < vals.length; j++) {
if (i != j && vals[j].label !== "NET")
isNET = val_arr.some(v => vals[j].array.indexOf(v) >= 0);
if (isNET)
break;
}
}
if (!isNET)
labels.push(vals[i].label);
}
return labels;
}
function readLevelsFromRFactor(v) {
const new_qname = "questionadffewe245";
const new_vname = "variableREWRRE12323112";
var data_file = v.question.dataFile;
// var expression = 'xxxxx122345xxx <- ' + stringToRName(question.name);
var expression = 'xxxxx122345foo <- ' + generateDisambiguatedVariableName(v);
try {
var new_r_question = data_file.newRQuestion(expression, new_qname, new_vname,
data_file.variables[data_file.variables.length - 1]);
// var r_output = group.appendR(expression);
//var levels = r_output.data.getAttribute([], "levels");
var value_attributes = new_r_question.valueAttributes;
var unique_values = new_r_question.uniqueValues;
var labels = unique_values.filter(u => !value_attributes.getIsMissingData(u)).map(u => value_attributes.getLabel(u));
var levels_excluding_hidden = determineNonNETLabels(v);
if (levels_excluding_hidden.length < labels.length)
labels = reorderHiddenCodes(labels, levels_excluding_hidden, v);
new_r_question.deleteQuestion();
return labels;
}catch(e){
log(e.message);
return determineNonNETLabels(v);
}
}
function reorderHiddenCodes(levels_fromR, levels_excluding_hidden, v) {
var value_attributes = v.question.valueAttributes;
var hidden_labels = levels_fromR.slice(levels_excluding_hidden.length, levels_fromR.length);
var hidden_vals = hidden_labels.map(l => value_attributes.getValue(value_attributes.getSourceValueByLabel(l)));
var dr = v.question.dataReduction;
var lvl_sourcevals = levels_fromR.map(function(l){ try {
var sv = [value_attributes.getSourceValueByLabel(l)];
}catch(e){ // merge
var sv = dr.getUnderlyingValues(l);
}
return sv;})
// Use mean of component values for merged categories
var lvl_vals = lvl_sourcevals.map(function(sv){
return sv.reduce((v1,v2) => value_attributes.getValue(v1)+value_attributes.getValue(v2),0)/sv.length;
});
var unhidden_lvl_vals = lvl_vals.slice(0, lvl_vals.length - hidden_vals.length);
// for each hidden val, determine where to splice into label array
var pos = 0;
var hv;
for (var i = 0; i < hidden_vals.length; i++) {
hv = hidden_vals[i];
for (var j = 0; j < unhidden_lvl_vals.length; j++) {
if (hv < unhidden_lvl_vals[j]) {
break;
}else
pos += 1;
}
unhidden_lvl_vals.splice(pos, 0, hv);
levels_excluding_hidden.splice(pos, 0, hidden_labels[i]);
pos = 0;
}
return levels_excluding_hidden;
}
function getDataFileFromDependants(deps) {
var data_file = null;
var dep;
for (let i = 0; i < deps.length; i++) {
dep = deps[i];
if (dep.type === "Question"){
data_file = dep.dataFile;
break;
}else if (dep.type === "Plot" || dep.type === "Table") {
data_file = dep.primary.dataFile;
break;
}else if (dep.type === "Variable") {
data_file = dep.question.dataFile;
break;
}else if (dep.type === "R Output") {
data_file = getDataFileFromDependants(dep.dependants());
if (data_file !== null)
break;
}
}
return data_file;
}
function createControlsAndFilter(control_type) {
includeWeb("QScript Selection Functions");
includeWeb("QScript Utility Functions");
includeWeb("QScript Functions to Generate Outputs");
includeWeb("QScript R Output Functions");
includeWeb("QScript Value Attributes Functions");
includeWeb("QScript Data Reduction Functions");
// On the web just take from what is selected.
var is_displayr = (!!Q.isOnTheWeb && Q.isOnTheWeb());
if (!is_displayr)
{
log("Sorry, this feature is only available in Displayr.");
return false;
}
var structure_name = is_displayr ? "variable set" : "question";
var is_cb = control_type === "Combobox";
var is_tb = control_type === "Textbox";
if (project.report.selectedRaw().length === 0) {
log("Please select the Table(s), Plot(s), and/or R Output(s) to apply the filter to.");
return false;
}else
var group = project.report.selectedRaw()[0].group;
var questions_in_tables = getQuestionsSelectedInTables();
// Remove empty tables
questions_in_tables = questions_in_tables.filter(function (q) { return q.question != null; });
// Need to extra work to determine the data file if the use has no Q Plots or Tables
// but only R Outputs selected
var sub_items = project.report.selectedRaw(); // group.subItems;
var types = sub_items.map(i => i.type);
var routput_selected = types.indexOf("R Output") > -1;
if (questions_in_tables.length === 0) {
if (project.dataFiles.length === 0) {
log("You must add a Data Set to use this feature.");
return false;
}else if (project.dataFiles.length === 1)
var data_file = project.dataFiles[0];
else {
// Try to get dataFile from dependents in the r outputs; otherwise, prompt user
var data_file;
if (routput_selected) {
var r_outputs = sub_items.filter(i => i.type === "R Output");
for (var i = 0; i < r_outputs.length; i++) {
data_file = getDataFileFromDependants(r_outputs[i].dependants());
if (data_file !== null)
break;
}
}
if (!data_file)
data_file = selectOneDataFile('Select the data file of the questions you wish to use for the filter.', project.dataFiles);
}
}else {
var data_file = questions_in_tables[0].question.dataFile;
// Make sure all questions are from the same data set
if (!questions_in_tables.map(function (q) { return q.question.dataFile.name; }).every(function (type) { return type == data_file.name; })) {
log("All of the selected outputs must be from the same data file.");
return false;
}
}
var allowed_types = is_tb ? ["Text", "Text - Multi"] : ["Pick One", "Pick One - Multi"];
var candidate_questions = getAllQuestionsByTypes([data_file], allowed_types);
if (candidate_questions.length == 0) {
log("No appropriate " + structure_name + "s found in the data file for the selected output.");
return false;
}
var selected_questions = [];
while (selected_questions.length === 0) {
selected_questions = selectManyQuestions("Select " + structure_name + "s to use for the filter:", candidate_questions, true).questions;
if (selected_questions.length === 0)
alert("Please select at least one Variable Set from the list to continue.");
}
var multiple_selection = (is_cb ? askYesNo("Should the user be allowed to select more than one category for each variable?") : true);
var ignore_case = (is_tb ? askYesNo("Ignore case (e.g., match \"dog\" with \"Dog\")?") : true);
/////////////////////////////////////////////////////////////////////////////////////
// Add Controls to page
var all_vars = getVariablesFromQuestions(selected_questions);
var n_boxes = all_vars.length;
var v = [];
var lbs = [];
let above;
const rowPad = 10;
const height = is_cb || is_tb ? 25 : 100;
var lb_names = [];
var var_names = [];
var control_lab = control_type.substring(0, control_type.length - 3) + " Box";
var control_name = control_type.substring(0, control_type.length - 3) + "Box";
for (i = 0; i < n_boxes; i++) {
v = all_vars[i];
var ctrl = group.appendControl(control_type);
if (is_tb) {
ctrl.text = "";
}else {
var labels = readLevelsFromRFactor(v);
ctrl.itemList = labels;
ctrl.selectedItems = labels;
ctrl.selectionMode = multiple_selection ? "MultipleSelection" : "SingleSelection";
}
// DS-3217: Control names must be unique, but some users move controls
// to a page master slide. These cannot be found via QScript/recursively
// searching project.report). Hence, resort to try-catch/while loop until
// a unique name is generated
let unique_name_found = false;
let duplicate_count = 0;
let ref_name = generateUniqueControlsName(cleanVariableName(control_name + v.label));
while (!unique_name_found) {
let ref_name_unique = ref_name + (duplicate_count > 0 ? ("." + duplicate_count.toString()) : "");
try {
ctrl.referenceName = ref_name_unique;
unique_name_found = true;
}catch(e){
duplicate_count++;
}
}
ctrl.name = ctrl.referenceName;
lb_names.push(ctrl.name);
var_names[i] = stringToRName(v.name);
lbs.push(ctrl);
ctrl.top = i == 0 ? 0 : above.top + height + rowPad;
ctrl.height = height;
ctrl.left = 0;
above = ctrl;
}
//////////////////////////////////////////////////////////////////////////////
// Create filter as a new R Question
var lab = control_lab + " Filter " + selected_questions.map(function(q) {
return q.variables.map(function(v) { return v.label;}).join(" + "); }).join(" + ");
var new_question_name = preventDuplicateQuestionName(data_file, lab);
var vname = control_name.toLowerCase() + "_filter_" +
selected_questions.map(function(q) {
return q.name.replace(/[^a-zA-Z0-9_@\#\$\\]/g, '_').toLowerCase();
}).join("_");
var new_var_name = preventDuplicateVariableName(data_file, vname);
var r_expr = "";
if (is_tb) {
r_expr = ("IGNORE_CASE = " + (ignore_case ? "TRUE\n" : "FALSE\n"));
r_expr += "caseFun <- if (IGNORE_CASE) tolower else I\n";
}
for (var i = 0; i < n_boxes; i++) {
r_expr += (is_tb ?
("grepl(caseFun(`" + lb_names[i] + "`), caseFun(" + generateDisambiguatedVariableName(all_vars[i]) + "), fixed = TRUE)") :
(all_vars[i].name + " %in% `" + lb_names[i] + "`"));
if (i < n_boxes - 1)
r_expr += " & ";
}
try {
var new_r_question = data_file.newRQuestion(r_expr, new_question_name, new_var_name, getLastVariable(all_vars));
} catch (e) {
function errorFun(e){
log("The filter could not be created for the selected " + structure_name + "s: " + e.message);
return false;
}
if (/(R code is ambiguous)|(There are [0-9]* cases in this result)/.test(e.message)){
r_expr = "";
if (is_tb) {
r_expr = ("IGNORE_CASE = " + (ignore_case ? "TRUE\n" : "FALSE\n"));
r_expr += "caseFun <- if (IGNORE_CASE) tolower else I\n";
}
for (var i = 0; i < n_boxes; i++) {
(r_expr += is_tb ?
("grepl(caseFun(`" + lb_names[i] + "`), caseFun(" + generateDisambiguatedVariableName(all_vars[i]) + "), fixed = TRUE)") :
(generateDisambiguatedVariableName(all_vars[i]) + " %in% `" + lb_names[i] + "`"));
if (i < n_boxes - 1)
r_expr += " & ";
}
try {
var new_r_question = data_file.newRQuestion(r_expr, new_question_name, new_var_name, getLastVariable(all_vars));
} catch(e){
return errorFun(e);
}
}else{
return errorFun(e);
}
}
new_r_question.isFilter = true;
new_r_question.isHidden = true;
new_r_question.questionType = "Pick One";
insertAtHoverButtonIfShown(new_r_question);
///////////////////////////////////////////////////////////////////////////////////
// Apply filter to the selected outputs
var allowed_types = ["Plot", "R Output", "Table"];
if (types.filter(function(t){return allowed_types.indexOf(t) === -1;}).length > 0)
log("Some selections were not a Chart, Table or R Output and the created filter has not been applied to them.");
for (var i = 0; i < sub_items.length; i++)
if (allowed_types.indexOf(types[i]) > -1 && sub_items[i].filters !== null)
sub_items[i].filters = sub_items[i].filters.concat(new_r_question.variables);
return true;
}
See also