Elasticsearch service/utils

Please be sure of your agent type and version and pick the right documentation accordingly.

Node.js

This is the documentation of the forest-express-sequelize and forest-express-mongoose Node.js agents that will soon reach end-of-support.

forest-express-sequelize v9 and forest-express-mongoose v9 are replaced by @forestadmin/agent v1.

Please check your agent type and version and read on or switch to the right documentation.

Ruby on Rails

Python

PHP

Elasticsearch service/utils

Connecting to Elasticsearch with a Custom Service

This service wraps the Elasticsearch Node.js client and provides the following implementation:

• Get a list of records (with Pagination and Filters handling)

• Get a simple record

• Create a record

• Update an existing record

• Delete a record

Prototype

const { Client } = require('@elastic/elasticsearch');

// Our own utils that transform ForestAdmin filters to Elasticsearch one
const { esTranslateFilter } = require('../utils/filter-translator');

class ElasticsearchHelper {
  // Allow to create a ElasticsearchHelper on your elastic index
  constructor({
    index,
    filterDefinition,
    mappingFunction,
    sort,
  });

  // Get a List of Records based on the query (page, filter, search, sort)
  functionSearch ({
    pageSize, page, filters, options,
  });

  // Get a Record by Id
  getRecord(recordId);

  // Create a Record in your Elasticsearch index
  createRecord(recordToCreate);

  // Update a Record in from your Elasticsearch index
  updateRecord(recordToUpdate);

  // Remove a by Id
  removeRecord(recordId);

  // Remove multiple Ids
  removeRecords(recordsIdsToDelete);
}

module.exports = ElasticsearchHelper;

Full implementation

You need to add Elasticsearch Node.js client to your projectnpm install @elastic/elasticsearch

Creating utils to convert Express query filters to Elasticsearch one

This utils takes an object representing a filter from the ForestAdmin UI and transforms it into a filter for Elasticsearch.

• Date filters

• Number filters

• Text filters

• Enum filters

Full implementation

utils/filter-translator.js

const { BaseOperatorDateParser } = require('forest-express-sequelize');
const moment = require('moment');
/**
 * @enum {string}
 */
const DATE_OPERATORS = {
  today: 'today',
  yesterday: 'yesterday',
  previous_week: 'previous_week',
  previous_month: 'previous_month',
  previous_quarter: 'previous_quarter',
  previous_year: 'previous_year',
  previous_week_to_date: 'previous_week_to_date',
  previous_month_to_date: 'previous_month_to_date',
  previous_quarter_to_date: 'previous_quarter_to_date',
  previous_year_to_date: 'previous_year_to_date',
  previous_x_days: 'previous_x_days',
  previous_x_days_to_date: 'previous_x_days_to_date',
  past: 'past',
  future: 'future',
  before_x_hours_ago: 'before_x_hours_ago',
  after_x_hours_ago: 'after_x_hours_ago',
};

/**
 * @typedef {{
 *  field: string;
 *  operator: 'before' | 'after' | 'equal' | 'not_equal' | 'present' | 'blank' | 'today'
 *    | 'yesterday' | 'previous_x_days' | 'previous_week' | 'previous_month' | 'previous_quarter'
 *    | 'previous_year' | 'previous_x_days_to_date' | 'previous_week_to_date'
 *    | 'previous_month_to_date' | 'previous_quarter_to_date' | 'previous_year_to_date'
 *    | 'past' | 'future' | 'before_x_hours_ago' | 'after_x_hours_ago'
 *  value: string | null;
 * }} OneFilter
 *
 * @typedef {{
 *  aggregator: 'and' | 'or'
 *  conditions: OneFilter[]
 * }} FilterCombination
 *
 * @typedef {{
 *  timezone: string
 * }} FilterOptions
 */

/**
 * @enum {string}
 */
const FIELD_DEFINITIONS = {
  date: 'date',
  keyword: 'keyword',
  text: 'text',
  number: 'number',
};

/**
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 * @param {FilterOptions} options
 */
function equal(fieldDefinition, oneFilter, options) {
  switch (fieldDefinition) {
    case FIELD_DEFINITIONS.date: {
      const date = moment.tz(oneFilter.value, options.timezone);

      return {
        term: {
          [oneFilter.field]: date.toISOString(),
        },
      };
    }

    case FIELD_DEFINITIONS.keyword:
    case FIELD_DEFINITIONS.text:
    case FIELD_DEFINITIONS.number: {
      return {
        terms: {
          [oneFilter.field]: Array.isArray(oneFilter.value)
            ? oneFilter.value
            : [oneFilter.value],
        },
      };
    }

    default:
      throw new Error('Invalid field type for operator equal');
  }
}

/**
 * @param {(fieldDefinition: FieldDefinition, oneFilter: OneFilter, options: FilterOptions) => any}
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 * @param {FilterOptions} options
 */
function not(mapper, fieldDefinition, oneFilter, options) {
  return {
    bool: {
      must_not: mapper(fieldDefinition, oneFilter, options),
    },
  };
}

/**
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 */
function present(fieldDefinition, oneFilter) {
  return {
    exists: {
      field: oneFilter.field,
    },
  };
}

/**
 * @param {'gt' | 'lt'} rangeOperator
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 * @param {FilterOptions} options
 */
function dateInRange(rangeOperator, fieldDefinition, oneFilter, options) {
  if (fieldDefinition !== FIELD_DEFINITIONS.date) {
    throw new Error('Invalid field type for operator after');
  }

  return {
    range: {
      [oneFilter.field]: {
        [rangeOperator]: oneFilter.value,
        time_zone: options.timezone,
      },
    },
  };
}

/**
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 */
function startsWith(fieldDefinition, oneFilter) {
  if (
    ![FIELD_DEFINITIONS.keyword, FIELD_DEFINITIONS.text].includes(
      fieldDefinition
    )
  ) {
    throw new Error('Unsupported operator starts_with');
  }

  return {
    wildcard: {
      [oneFilter.field]: {
        value: `${oneFilter.value}*`,
        case_insensitive: true,
      },
    },
  };
}

/**
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 */
function endsWith(fieldDefinition, oneFilter) {
  if (
    ![FIELD_DEFINITIONS.keyword, FIELD_DEFINITIONS.text].includes(
      fieldDefinition
    )
  ) {
    throw new Error('Unsupported operator ends_with');
  }

  return {
    wildcard: {
      [oneFilter.field]: {
        value: `*${oneFilter.value}`,
        case_insensitive: true,
      },
    },
  };
}

/**
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 */
function contains(fieldDefinition, oneFilter) {
  if (
    ![FIELD_DEFINITIONS.keyword, FIELD_DEFINITIONS.text].includes(
      fieldDefinition
    )
  ) {
    throw new Error('Unsupported operator contains');
  }

  return {
    wildcard: {
      [oneFilter.field]: {
        value: `*${oneFilter.value}*`,
        case_insensitive: true,
      },
    },
  };
}

/**
 * @param {FieldDefinition} fieldDefinition
 * @param {OneFilter} oneFilter
 */
function numberInRange(rangeOperator, fieldDefinition, oneFilter) {
  if (![FIELD_DEFINITIONS.number].includes(fieldDefinition)) {
    throw new Error(`Unsupported operator ${rangeOperator}`);
  }

  return {
    range: {
      [oneFilter.field]: {
        [rangeOperator]: oneFilter.value,
      },
    },
  };
}

/**
 * @param {BaseOperatorDateParser} operatorDateParser
 * @param {OneFilter} filter
 * @param {FilterOptions} options
 * @returns { range: any}
 */
function mapDateOperator(operatorDateParser, filter, options) {
  return {
    range: {
      [filter.field]: {
        ...operatorDateParser.getDateFilter(filter.operator, filter.value),
        time_zone: options.timezone,
      },
    },
  };
}

const MAPPING = {
  equal,
  not_equal: not.bind(undefined, equal),
  present,
  blank: not.bind(undefined, present),
  before: dateInRange.bind(undefined, 'lt'),
  after: dateInRange.bind(undefined, 'gt'),
  starts_with: startsWith,
  ends_with: endsWith,
  contains,
  not_contains: not.bind(undefined, contains),
  greater_than: numberInRange.bind(undefined, 'gt'),
  less_than: numberInRange.bind(undefined, 'lt'),
};

/**
 * @param {Record<string, FIELD_DEFINITIONS>} fieldDefinitions
 * @param {BaseOperatorDateParser} operatorDateParser
 * @param {FilterOptions} options
 * @param {OneFilter} oneFilter
 */
function mapFilter(fieldDefinitions, operatorDateParser, options, oneFilter) {
  if (
    fieldDefinitions[oneFilter.field] === FIELD_DEFINITIONS.date &&
    Object.values(DATE_OPERATORS).includes(oneFilter.operator)
  ) {
    return mapDateOperator(operatorDateParser, oneFilter, options);
  }

  const mapper = MAPPING[oneFilter.operator];
  const fieldDefinition = fieldDefinitions[oneFilter.field];

  if (!mapper) {
    throw new Error(
      `Unknown operator ${oneFilter.operator}, you need to define it !`
    );
  }
  if (!fieldDefinition) {
    throw new Error(
      `Unknown field ${oneFilter.field}, your field hasn't any field definition. Please check your ElasticsearchHelper configuration`
    );
  }

  return mapper(fieldDefinition, oneFilter, options);
}

/**
 * Takes an object representing a filter from the UI
 * and transforms it into a filter for elasticSearch
 * @param {Record<string, FIELD_DEFINITIONS>} fieldDefinitions
 * @param {FilterCombination | OneFilter} filters
 * @param {FilterOptions} options
 * @returns {any} A valid ES filter
 */
function esTranslateFilter(fieldDefinitions, filters, options) {
  if (!filters) {
    return [];
  }

  const operatorDateParser = new BaseOperatorDateParser({
    timezone: options.timezone,
    operators: {
      GTE: 'gte',
      LTE: 'lte',
      GT: 'gt',
      LT: 'lt',
    },
  });

  if (!filters.aggregator) {
    return [mapFilter(fieldDefinitions, operatorDateParser, options, filters)];
  }

  const mapped = filters.conditions.map(
    mapFilter.bind(undefined, fieldDefinitions, operatorDateParser, options)
  );

  if (filters.aggregator === 'and') {
    return mapped;
  }

  return {
    bool: {
      should: mapped,
      minimum_should_match: 1,
    },
  };
}

exports.esTranslateFilter = esTranslateFilter;
exports.FIELD_DEFINITIONS = FIELD_DEFINITIONS;

We expose utils to parse filters through forest-express-sequelize since version 7.6.0