GoodDataFile: src/execution/experimental-executions.ts
// (C) 2007-2020 GoodData Corporation
import md5 from "md5";
import invariant from "invariant";
import cloneDeep from "lodash/cloneDeep";
import compact from "lodash/compact";
import filter from "lodash/filter";
import first from "lodash/first";
import find from "lodash/find";
import map from "lodash/map";
import merge from "lodash/merge";
import every from "lodash/every";
import get from "lodash/get";
import isEmpty from "lodash/isEmpty";
import negate from "lodash/negate";
import partial from "lodash/partial";
import flatten from "lodash/flatten";
import set from "lodash/set";
import { Rules } from "../utils/rules";
import { sortDefinitions } from "../utils/definitions";
import { getMissingUrisInAttributesMap } from "../utils/attributesMapLoader";
import {
getAttributes,
getAttributesDisplayForms,
getDefinition,
getMeasureFilters,
getMeasures,
isAttributeMeasureFilter,
} from "../utils/visualizationObjectHelper";
import { IMeasure } from "../interfaces";
import { XhrModule } from "../xhr";
const notEmpty = negate(isEmpty);
function findHeaderForMappingFn(mapping: any, header: any) {
return (
(mapping.element === header.id || mapping.element === header.uri) && header.measureIndex === undefined
);
}
function wrapMeasureIndexesFromMappings(metricMappings: any[], headers: any[]) {
if (metricMappings) {
metricMappings.forEach(mapping => {
const header = find(headers, partial(findHeaderForMappingFn, mapping));
if (header) {
header.measureIndex = mapping.measureIndex;
header.isPoP = mapping.isPoP;
}
});
}
return headers;
}
const emptyResult = {
extendedTabularDataResult: {
values: [],
warnings: [],
},
};
const MAX_TITLE_LENGTH = 1000;
function getMetricTitle(suffix: string, title: string) {
const maxLength = MAX_TITLE_LENGTH - suffix.length;
if (title && title.length > maxLength) {
if (title[title.length - 1] === ")") {
return `${title.substring(0, maxLength - 2)}…)${suffix}`;
}
return `${title.substring(0, maxLength - 1)}…${suffix}`;
}
return `${title}${suffix}`;
}
const getBaseMetricTitle = partial(getMetricTitle, "");
const CONTRIBUTION_METRIC_FORMAT = "#,##0.00%";
function getPoPDefinition(measure: IMeasure) {
return get(measure, ["definition", "popMeasureDefinition"], {});
}
function getAggregation(measure: IMeasure) {
return get(getDefinition(measure), "aggregation", "").toLowerCase();
}
function isEmptyFilter(metricFilter: any) {
if (get(metricFilter, "positiveAttributeFilter")) {
return isEmpty(get(metricFilter, ["positiveAttributeFilter", "in"]));
}
if (get(metricFilter, "negativeAttributeFilter")) {
return isEmpty(get(metricFilter, ["negativeAttributeFilter", "notIn"]));
}
if (get(metricFilter, "absoluteDateFilter")) {
return (
get(metricFilter, ["absoluteDateFilter", "from"]) === undefined &&
get(metricFilter, ["absoluteDateFilter", "to"]) === undefined
);
}
return (
get(metricFilter, ["relativeDateFilter", "from"]) === undefined &&
get(metricFilter, ["relativeDateFilter", "to"]) === undefined
);
}
function allFiltersEmpty(item: any) {
return every(map(getMeasureFilters(item), f => isEmptyFilter(f)));
}
function isDerived(measure: any) {
const aggregation = getAggregation(measure);
return aggregation !== "" || !allFiltersEmpty(measure);
}
function getAttrTypeFromMap(dfUri: string, attributesMap: any) {
return get(get(attributesMap, [dfUri], {}), ["attribute", "content", "type"]);
}
function getAttrUriFromMap(dfUri: string, attributesMap: any) {
return get(get(attributesMap, [dfUri], {}), ["attribute", "meta", "uri"]);
}
function isAttrFilterNegative(attributeFilter: any) {
return get(attributeFilter, "negativeAttributeFilter") !== undefined;
}
function getAttrFilterElements(attributeFilter: any) {
const isNegative = isAttrFilterNegative(attributeFilter);
const pathToElements = isNegative
? ["negativeAttributeFilter", "notIn"]
: ["positiveAttributeFilter", "in"];
return get(attributeFilter, pathToElements, []);
}
function getAttrFilterExpression(measureFilter: any, attributesMap: any) {
const isNegative = get(measureFilter, "negativeAttributeFilter", false);
const detailPath = isNegative ? "negativeAttributeFilter" : "positiveAttributeFilter";
const attributeUri = getAttrUriFromMap(
get(measureFilter, [detailPath, "displayForm", "uri"]),
attributesMap,
);
const elements = getAttrFilterElements(measureFilter);
if (isEmpty(elements)) {
return null;
}
const elementsForQuery = map(elements, e => `[${e}]`);
const negative = isNegative ? "NOT " : "";
return `[${attributeUri}] ${negative}IN (${elementsForQuery.join(",")})`;
}
function getDateFilterExpression() {
// measure date filter was never supported
return "";
}
function getFilterExpression(attributesMap: any, measureFilter: any) {
if (isAttributeMeasureFilter(measureFilter)) {
return getAttrFilterExpression(measureFilter, attributesMap);
}
return getDateFilterExpression();
}
function getGeneratedMetricExpression(item: any, attributesMap: any) {
const aggregation = getAggregation(item).toUpperCase();
const objectUri = get(getDefinition(item), "item.uri");
const where = filter(map(getMeasureFilters(item), partial(getFilterExpression, attributesMap)), e => !!e);
return `SELECT ${aggregation ? `${aggregation}([${objectUri}])` : `[${objectUri}]`}${
notEmpty(where) ? ` WHERE ${where.join(" AND ")}` : ""
}`;
}
function getPercentMetricExpression(category: any, attributesMap: any, measure: any) {
let metricExpressionWithoutFilters = `SELECT [${get(getDefinition(measure), "item.uri")}]`;
if (isDerived(measure)) {
metricExpressionWithoutFilters = getGeneratedMetricExpression(
set(cloneDeep(measure), ["definition", "measureDefinition", "filters"], []),
attributesMap,
);
}
const attributeUri = getAttrUriFromMap(get(category, "displayForm.uri"), attributesMap);
const whereFilters = filter(
map(getMeasureFilters(measure), partial(getFilterExpression, attributesMap)),
e => !!e,
);
const whereExpression = notEmpty(whereFilters) ? ` WHERE ${whereFilters.join(" AND ")}` : "";
// tslint:disable-next-line:max-line-length
return `SELECT (${metricExpressionWithoutFilters}${whereExpression}) / (${metricExpressionWithoutFilters} BY ALL [${attributeUri}]${whereExpression})`;
}
function getPoPExpression(attributeUri: string, metricExpression: string) {
return `SELECT ${metricExpression} FOR PREVIOUS ([${attributeUri}])`;
}
function getGeneratedMetricHash(title: string, format: string, expression: string) {
return md5(`${expression}#${title}#${format}`);
}
function getMeasureType(measure: any) {
const aggregation = getAggregation(measure);
if (aggregation === "") {
return "metric";
} else if (aggregation === "count") {
return "attribute";
}
return "fact";
}
function getGeneratedMetricIdentifier(
item: any,
aggregation: string,
expressionCreator: (item: any, attributesMap: any) => string,
hasher: any,
attributesMap: any,
) {
const [, , , prjId, , id] = get(getDefinition(item), "item.uri", "").split("/");
const identifier = `${prjId}_${id}`;
const hash = hasher(expressionCreator(item, attributesMap));
const hasNoFilters = isEmpty(getMeasureFilters(item));
const type = getMeasureType(item);
const prefix = hasNoFilters || allFiltersEmpty(item) ? "" : "_filtered";
return `${type}_${identifier}.generated.${hash}${prefix}_${aggregation}`;
}
function isDateAttribute(attribute: any, attributesMap = {}) {
return getAttrTypeFromMap(get(attribute, ["displayForm", "uri"]), attributesMap) !== undefined;
}
function getMeasureSorting(measure?: any, mdObj?: any) {
const sorting = get(mdObj, ["properties", "sortItems"], []);
const matchedSorting = sorting.find((sortItem: any) => {
const measureSortItem = get(sortItem, ["measureSortItem"]);
if (measureSortItem) {
// only one item now, we support only 2d data
const identifier = get(measureSortItem, [
"locators",
0,
"measureLocatorItem",
"measureIdentifier",
]);
return identifier === get(measure, "localIdentifier");
}
return false;
});
if (matchedSorting) {
return get(matchedSorting, ["measureSortItem", "direction"], null);
}
return null;
}
function getCategorySorting(category: any, mdObj: any) {
const sorting = get(mdObj, ["properties", "sortItems"], []);
const matchedSorting = sorting.find((sortItem: any) => {
const attributeSortItem = get(sortItem, ["attributeSortItem"]);
if (attributeSortItem) {
const identifier = get(attributeSortItem, ["attributeIdentifier"]);
return identifier === get(category, "localIdentifier");
}
return false;
});
if (matchedSorting) {
return get(matchedSorting, ["attributeSortItem", "direction"], null);
}
return null;
}
const createPureMetric = (measure: any, mdObj: any, measureIndex: number) => ({
element: get(measure, ["definition", "measureDefinition", "item", "uri"]),
sort: getMeasureSorting(measure, mdObj),
meta: { measureIndex },
});
function createDerivedMetric(measure: any, mdObj: any, measureIndex: number, attributesMap: any) {
const { format } = measure;
const sort = getMeasureSorting(measure, mdObj);
const title = getBaseMetricTitle(measure.title);
const hasher = partial(getGeneratedMetricHash, title, format);
const aggregation = getAggregation(measure);
const element = getGeneratedMetricIdentifier(
measure,
aggregation.length ? aggregation : "base",
getGeneratedMetricExpression,
hasher,
attributesMap,
);
const definition = {
metricDefinition: {
identifier: element,
expression: getGeneratedMetricExpression(measure, attributesMap),
title,
format,
},
};
return {
element,
definition,
sort,
meta: {
measureIndex,
},
};
}
function createContributionMetric(measure: any, mdObj: any, measureIndex: number, attributesMap: any) {
const attribute = first(getAttributes(mdObj));
const getMetricExpression = partial(getPercentMetricExpression, attribute, attributesMap);
const title = getBaseMetricTitle(get(measure, "title"));
const hasher = partial(getGeneratedMetricHash, title, CONTRIBUTION_METRIC_FORMAT);
const identifier = getGeneratedMetricIdentifier(
measure,
"percent",
getMetricExpression,
hasher,
attributesMap,
);
return {
element: identifier,
definition: {
metricDefinition: {
identifier,
expression: getMetricExpression(measure),
title,
format: CONTRIBUTION_METRIC_FORMAT,
},
},
sort: getMeasureSorting(measure, mdObj),
meta: {
measureIndex,
},
};
}
function getOriginalMeasureForPoP(popMeasure: any, mdObj: any) {
return getMeasures(mdObj).find(
(measure: any) =>
get(measure, "localIdentifier") === get(getPoPDefinition(popMeasure), ["measureIdentifier"]),
);
}
function createPoPMetric(popMeasure: any, mdObj: any, measureIndex: number, attributesMap: any) {
const title = getBaseMetricTitle(get(popMeasure, "title"));
const format = get(popMeasure, "format");
const hasher = partial(getGeneratedMetricHash, title, format);
const attributeUri = get(popMeasure, "definition.popMeasureDefinition.popAttribute.uri");
const originalMeasure = getOriginalMeasureForPoP(popMeasure, mdObj);
const originalMeasureExpression = `[${get(getDefinition(originalMeasure), ["item", "uri"])}]`;
let metricExpression = getPoPExpression(attributeUri, originalMeasureExpression);
if (isDerived(originalMeasure)) {
const generated = createDerivedMetric(originalMeasure, mdObj, measureIndex, attributesMap);
const generatedMeasureExpression = `(${get(generated, [
"definition",
"metricDefinition",
"expression",
])})`;
metricExpression = getPoPExpression(attributeUri, generatedMeasureExpression);
}
const identifier = getGeneratedMetricIdentifier(
originalMeasure,
"pop",
() => metricExpression,
hasher,
attributesMap,
);
return {
element: identifier,
definition: {
metricDefinition: {
identifier,
expression: metricExpression,
title,
format,
},
},
sort: getMeasureSorting(popMeasure, mdObj),
meta: {
measureIndex,
isPoP: true,
},
};
}
function createContributionPoPMetric(popMeasure: any, mdObj: any, measureIndex: number, attributesMap: any) {
const attributeUri = get(popMeasure, ["definition", "popMeasureDefinition", "popAttribute", "uri"]);
const originalMeasure = getOriginalMeasureForPoP(popMeasure, mdObj);
const generated = createContributionMetric(originalMeasure, mdObj, measureIndex, attributesMap);
const title = getBaseMetricTitle(get(popMeasure, "title"));
const format = CONTRIBUTION_METRIC_FORMAT;
const hasher = partial(getGeneratedMetricHash, title, format);
const generatedMeasureExpression = `(${get(generated, [
"definition",
"metricDefinition",
"expression",
])})`;
const metricExpression = getPoPExpression(attributeUri, generatedMeasureExpression);
const identifier = getGeneratedMetricIdentifier(
originalMeasure,
"pop",
() => metricExpression,
hasher,
attributesMap,
);
return {
element: identifier,
definition: {
metricDefinition: {
identifier,
expression: metricExpression,
title,
format,
},
},
sort: getMeasureSorting(),
meta: {
measureIndex,
isPoP: true,
},
};
}
function categoryToElement(attributesMap: any, mdObj: any, category: any) {
const element = getAttrUriFromMap(get(category, ["displayForm", "uri"]), attributesMap);
return {
element,
sort: getCategorySorting(category, mdObj),
};
}
function isPoP({ definition }: any) {
return get(definition, "popMeasureDefinition") !== undefined;
}
function isContribution({ definition }: any) {
return get(definition, ["measureDefinition", "computeRatio"]);
}
function isPoPContribution(popMeasure: any, mdObj: any) {
if (isPoP(popMeasure)) {
const originalMeasure = getOriginalMeasureForPoP(popMeasure, mdObj);
return isContribution(originalMeasure);
}
return false;
}
function isCalculatedMeasure({ definition }: any) {
return get(definition, ["measureDefinition", "aggregation"]) === undefined;
}
const rules = new Rules();
rules.addRule([isPoPContribution], createContributionPoPMetric);
rules.addRule([isPoP], createPoPMetric);
rules.addRule([isContribution], createContributionMetric);
rules.addRule([isDerived], createDerivedMetric);
rules.addRule([isCalculatedMeasure], createPureMetric);
function getMetricFactory(measure: any, mdObj: any) {
const factory = rules.match(measure, mdObj);
invariant(factory, `Unknown factory for: ${measure}`);
return factory;
}
function getExecutionDefinitionsAndColumns(mdObj: any, options: any, attributesMap: any) {
const measures = getMeasures(mdObj);
let attributes = getAttributes(mdObj);
const metrics = flatten(
map(measures, (measure, index) =>
getMetricFactory(measure, mdObj)(measure, mdObj, index, attributesMap),
),
);
if (options.removeDateItems) {
attributes = filter(attributes, attribute => !isDateAttribute(attribute, attributesMap));
}
attributes = map(attributes, partial(categoryToElement, attributesMap, mdObj));
const columns = compact(map([...attributes, ...metrics], "element"));
return {
columns,
definitions: sortDefinitions(compact(map(metrics, "definition"))),
};
}
/**
* Module for execution on experimental execution resource
*
* @class execution
* @module execution
* @deprecated The module is in maintenance mode only (just the the compilation issues are being fixed when
* referenced utilities and interfaces are being changed) and is not being extended when AFM executor
* have new functionality added.
*/
export class ExperimentalExecutionsModule {
constructor(private xhr: XhrModule, private loadAttributesMap: any) {}
/**
* For the given projectId it returns table structure with the given
* elements in column headers.
*
* @method getData
* @param {String} projectId - GD project identifier
* @param {Array} columns - An array of attribute or metric identifiers.
* @param {Object} executionConfiguration - Execution configuration - can contain for example
* property "where" containing query-like filters
* property "orderBy" contains array of sorted properties to order in form
* [{column: 'identifier', direction: 'asc|desc'}]
* @param {Object} settings - Supports additional settings accepted by the underlying
* xhr.ajax() calls
*
* @return {Object} Structure with `headers` and `rawData` keys filled with values from execution.
*/
public getData(projectId: string, columns: any[], executionConfiguration: any = {}, settings: any = {}) {
if (process.env.NODE_ENV !== "test") {
// tslint:disable-next-line:no-console
console.warn(
"ExperimentalExecutionsModule is deprecated and is no longer being maintained. " +
"Please migrate to the ExecuteAfmModule.",
);
}
const executedReport: any = {
isLoaded: false,
};
// Create request and result structures
const request: any = {
execution: { columns },
};
// enrich configuration with supported properties such as
// where clause with query-like filters
["where", "orderBy", "definitions"].forEach(property => {
if (executionConfiguration[property]) {
request.execution[property] = executionConfiguration[property];
}
});
// Execute request
return this.xhr
.post(`/gdc/internal/projects/${projectId}/experimental/executions`, {
...settings,
body: JSON.stringify(request),
})
.then(r => r.getData())
.then(response => {
executedReport.headers = wrapMeasureIndexesFromMappings(
get(executionConfiguration, "metricMappings"),
get(response, ["executionResult", "headers"], []),
);
// Start polling on url returned in the executionResult for tabularData
return this.loadExtendedDataResults(
response.executionResult.extendedTabularDataResult,
settings,
);
})
.then((r: any) => {
const { result, status } = r;
return {
...executedReport,
rawData: get(result, "extendedTabularDataResult.values", []),
warnings: get(result, "extendedTabularDataResult.warnings", []),
isLoaded: true,
isEmpty: status === 204,
};
});
}
public mdToExecutionDefinitionsAndColumns(projectId: string, mdObj: any, options = {}) {
const allDfUris = getAttributesDisplayForms(mdObj);
const attributesMapPromise = this.getAttributesMap(options, allDfUris, projectId);
return attributesMapPromise.then((attributesMap: any) => {
return getExecutionDefinitionsAndColumns(mdObj, options, attributesMap);
});
}
private getAttributesMap(options: any, displayFormUris: string[], projectId: string) {
const attributesMap = get(options, "attributesMap", {});
const missingUris = getMissingUrisInAttributesMap(displayFormUris, attributesMap);
return this.loadAttributesMap(projectId, missingUris).then((result: any) => {
return {
...attributesMap,
...result,
};
});
}
private loadExtendedDataResults(uri: string, settings: any, prevResult = emptyResult) {
return new Promise((resolve, reject) => {
this.xhr
.ajax(uri, settings)
.then(r => {
const { response } = r;
if (response.status === 204) {
return {
status: response.status,
result: "",
};
}
return {
status: response.status,
result: r.getData(),
};
})
.then(({ status, result }) => {
const values = [
...get(prevResult, "extendedTabularDataResult.values", []),
...get(result, "extendedTabularDataResult.values", []),
];
const warnings = [
...get(prevResult, "extendedTabularDataResult.warnings", []),
...get(result, "extendedTabularDataResult.warnings", []),
];
const updatedResult = merge({}, prevResult, {
extendedTabularDataResult: {
values,
warnings,
},
});
const nextUri = get(result, "extendedTabularDataResult.paging.next");
if (nextUri) {
resolve(this.loadExtendedDataResults(nextUri, settings, updatedResult));
} else {
resolve({ status, result: updatedResult });
}
}, reject);
});
}
}