/*
 * (c) Copyright Ascensio System SIA 2010-2024
 *
 * This program is a free software product. You can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License (AGPL)
 * version 3 as published by the Free Software Foundation. In accordance with
 * Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
 * that Ascensio System SIA expressly excludes the warranty of non-infringement
 * of any third-party rights.
 *
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE. For
 * details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
 *
 * You can contact Ascensio System SIA at 20A-6 Ernesta Birznieka-Upish
 * street, Riga, Latvia, EU, LV-1050.
 *
 * The  interactive user interfaces in modified source and object code versions
 * of the Program must display Appropriate Legal Notices, as required under
 * Section 5 of the GNU AGPL version 3.
 *
 * Pursuant to Section 7(b) of the License you must retain the original Product
 * logo when distributing the program. Pursuant to Section 7(e) we decline to
 * grant you any rights under trademark law for use of our trademarks.
 *
 * All the Product's GUI elements, including illustrations and icon sets, as
 * well as technical writing content are licensed under the terms of the
 * Creative Commons Attribution-ShareAlike 4.0 International. See the License
 * terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
 *
 */

'use strict';
const os = require('os');
const path = require('path');
const fs = require('fs');
const url = require('url');
const co = require('co');
const config = require('config');
const spawnAsync = require('@expo/spawn-async');
const bytes = require('bytes');
const lcid = require('lcid');
const ms = require('ms');
const {pipeline} = require('node:stream/promises');

const commonDefines = require('./../../Common/sources/commondefines');
const storage = require('./../../Common/sources/storage/storage-base');
const utils = require('./../../Common/sources/utils');
const constants = require('./../../Common/sources/constants');
const baseConnector = require('../../DocService/sources/databaseConnectors/baseConnector');
const wopiUtils = require('./../../DocService/sources/wopiUtils');
const taskResult = require('./../../DocService/sources/taskresult');
const statsDClient = require('./../../Common/sources/statsdclient');
const queueService = require('./../../Common/sources/taskqueueRabbitMQ');
const formatChecker = require('./../../Common/sources/formatchecker');
const operationContext = require('./../../Common/sources/operationContext');
const tenantManager = require('./../../Common/sources/tenantManager');
const {detectCertType} = require('./signing/pdfSigningCore');
const {signPdfFile: signPdfFileKms} = require('./signing/pdfAwsKmsSigner');
const {signPdfFile: signPdfFileCsc} = require('./signing/pdfCscSigner');

const cfgMaxDownloadBytes = config.get('FileConverter.converter.maxDownloadBytes');
const cfgDownloadTimeout = config.get('FileConverter.converter.downloadTimeout');
const cfgDownloadAttemptMaxCount = config.get('FileConverter.converter.downloadAttemptMaxCount');
const cfgDownloadAttemptDelay = config.get('FileConverter.converter.downloadAttemptDelay');
const cfgFontDir = config.get('FileConverter.converter.fontDir');
const cfgPresentationThemesDir = config.get('FileConverter.converter.presentationThemesDir');
const cfgX2tPath = config.get('FileConverter.converter.x2tPath');
const cfgDocbuilderPath = config.get('FileConverter.converter.docbuilderPath');
const cfgArgs = config.get('FileConverter.converter.args');
const cfgSpawnOptions = config.util.cloneDeep(config.get('FileConverter.converter.spawnOptions'));
const cfgErrorFiles = config.get('FileConverter.converter.errorfiles');
const cfgInputLimits = config.get('FileConverter.converter.inputLimits');
const cfgStreamWriterBufferSize = config.get('FileConverter.converter.streamWriterBufferSize');
const cfgSigningKeyStorePath =
  config.get('FileConverter.converter.signing.keyStorePath') || config.get('FileConverter.converter.signingKeyStorePath');
const cfgSigning = config.get('FileConverter.converter.signing');

/**
 * @param {Object} ctx
 * @returns {string} resolved signing certificate path (new or legacy)
 */
function resolveSigningPath(ctx) {
  return (
    ctx.getCfg('FileConverter.converter.signing.keyStorePath', null) ||
    ctx.getCfg('FileConverter.converter.signingKeyStorePath', cfgSigningKeyStorePath) ||
    ''
  );
}

/**
 * Merge signing metadata from config into jsonParams as pdfLayout.signature.
 * @param {Object} signingCfg - FileConverter.converter.signing config
 * @param {string} jsonParams - existing JSON-encoded params (may be empty/null)
 * @returns {string} updated JSON-encoded params
 */
function mergeSigningMeta(signingCfg, jsonParams) {
  const meta = signingCfg?.meta;
  if (!meta || (!meta.reason && !meta.name && !meta.location && !meta.contactInfo)) return jsonParams;
  const parsed = jsonParams ? JSON.parse(jsonParams) : {};
  parsed.pdfLayout = {...parsed.pdfLayout, signature: meta};
  return JSON.stringify(parsed);
}

/**
 * @param {string} certPath - resolved signing certificate path
 * @param {Object} signingCfg - FileConverter.converter.signing config
 * @returns {{isCloud: boolean, certPath: string}}
 */
function getCloudSigningMode(certPath, signingCfg) {
  if (!certPath || !fs.existsSync(certPath)) return {isCloud: false, certPath};
  const hasCloudProvider = !!(signingCfg?.awsKms?.keyId || signingCfg?.csc?.baseUrl);
  if (!hasCloudProvider) return {isCloud: false, certPath};

  return {isCloud: detectCertType(certPath) === 'pem', certPath};
}

/**
 * @param {Object} ctx
 * @param {string} outputPath - PDF file to sign
 * @param {string} certPath - PEM chain path
 * @param {Object} signingCfg - FileConverter.converter.signing config
 * @returns {Promise<void>}
 */
function performCloudSigning(ctx, outputPath, certPath, signingCfg) {
  const awsKmsCfg = signingCfg?.awsKms || {};
  if (awsKmsCfg.keyId) {
    ctx.logger.debug('Cloud signing (AWS KMS) start');
    return signPdfFileKms(outputPath, null, {
      keyId: awsKmsCfg.keyId,
      endpoint: awsKmsCfg.endpoint,
      accessKeyId: awsKmsCfg.accessKeyId,
      secretAccessKey: awsKmsCfg.secretAccessKey,
      keyStorePath: certPath
    });
  }
  const cscCfg = signingCfg?.csc || {};
  if (cscCfg.baseUrl) {
    ctx.logger.debug('Cloud signing (CSC) start');
    return signPdfFileCsc(outputPath, null, {
      ...cscCfg,
      keyStorePath: certPath
    });
  }
  return Promise.reject(new Error('No cloud signing provider configured'));
}
//cfgMaxRequestChanges was obtained as a result of the test: 84408 changes - 5,16 MB
const cfgMaxRequestChanges = config.get('services.CoAuthoring.server.maxRequestChanges');
const cfgForgottenFiles = config.get('services.CoAuthoring.server.forgottenfiles');
const cfgForgottenFilesName = config.get('services.CoAuthoring.server.forgottenfilesname');
const cfgNewFileTemplate = config.get('services.CoAuthoring.server.newFileTemplate');
const cfgEditor = config.get('services.CoAuthoring.editor');
const cfgRequesFilteringAgent = config.util.cloneDeep(config.get('services.CoAuthoring.request-filtering-agent'));
const cfgExternalRequestDirectIfIn = config.get('externalRequest.directIfIn');
const cfgExternalRequestAction = config.get('externalRequest.action');

//windows limit 512(2048) https://msdn.microsoft.com/en-us/library/6e3b887c.aspx
//Ubuntu 14.04 limit 4096 http://underyx.me/2015/05/18/raising-the-maximum-number-of-file-descriptors.html
//MacOs limit 2048 http://apple.stackexchange.com/questions/33715/too-many-open-files
const MAX_OPEN_FILES = 200;
const TEMP_PREFIX = 'ASC_CONVERT';
let queue = null;
const clientStatsD = statsDClient.getClient();
const exitCodesReturn = [
  constants.CONVERT_PARAMS,
  constants.CONVERT_NEED_PARAMS,
  constants.CONVERT_CORRUPTED,
  constants.CONVERT_DRM,
  constants.CONVERT_DRM_UNSUPPORTED,
  constants.CONVERT_PASSWORD,
  constants.CONVERT_LIMITS,
  constants.CONVERT_DETECT
];
const exitCodesMinorError = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM, constants.CONVERT_DRM_UNSUPPORTED, constants.CONVERT_PASSWORD];
const exitCodesUpload = [
  constants.NO_ERROR,
  constants.CONVERT_CORRUPTED,
  constants.CONVERT_NEED_PARAMS,
  constants.CONVERT_DRM,
  constants.CONVERT_DRM_UNSUPPORTED
];
const exitCodesCopyOrigin = [constants.CONVERT_NEED_PARAMS, constants.CONVERT_DRM];

function TaskQueueDataConvert(ctx, task) {
  const cmd = task.getCmd();
  this.key = cmd.getDocId();
  if (cmd.getSaveKey()) {
    this.key += cmd.getSaveKey();
  }
  this.fileFrom = null;
  this.fileTo = null;
  this.title = cmd.getTitle();
  if (constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA !== cmd.getOutputFormat()) {
    this.formatTo = cmd.getOutputFormat();
  } else {
    this.formatTo = constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF;
    this.isPDFA = true;
  }
  this.csvTxtEncoding = cmd.getCodepage();
  this.csvDelimiter = cmd.getDelimiter();
  this.csvDelimiterChar = cmd.getDelimiterChar();
  this.paid = task.getPaid();
  this.embeddedFonts = cmd.embeddedfonts;
  this.fromChanges = task.getFromChanges();
  //todo
  const tenFontDir = ctx.getCfg('FileConverter.converter.fontDir', cfgFontDir);
  if (tenFontDir) {
    this.fontDir = path.resolve(tenFontDir);
  } else {
    this.fontDir = null;
  }
  const tenPresentationThemesDir = ctx.getCfg('FileConverter.converter.presentationThemesDir', cfgPresentationThemesDir);
  this.themeDir = path.resolve(tenPresentationThemesDir);
  this.mailMergeSend = cmd.mailmergesend;
  this.thumbnail = cmd.thumbnail;
  this.textParams = cmd.getTextParams();
  this.jsonParams = JSON.stringify(cmd.getJsonParams());
  this.lcid = cmd.getLCID();
  this.password = cmd.getPassword();
  this.savePassword = cmd.getSavePassword();
  this.noBase64 = cmd.getNoBase64();
  this.convertToOrigin = cmd.getConvertToOrigin();
  this.oformAsPdf = cmd.getOformAsPdf();
  const forceSave = cmd.getForceSave();
  this.forceSaveType = forceSave?.getType();
  this.timestamp = new Date();
}
TaskQueueDataConvert.prototype = {
  serialize(ctx, fsPath) {
    let xml = '\ufeff<?xml version="1.0" encoding="utf-8"?>';
    xml += '<TaskQueueDataConvert xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"';
    xml += ' xmlns:xsd="http://www.w3.org/2001/XMLSchema">';
    xml += this.serializeXmlProp('m_sKey', this.key);
    xml += this.serializeXmlProp('m_sFileFrom', this.fileFrom);
    xml += this.serializeXmlProp('m_sFileTo', this.fileTo);
    xml += this.serializeXmlProp('m_sTitle', this.title);
    xml += this.serializeXmlProp('m_nFormatTo', this.formatTo);
    xml += this.serializeXmlProp('m_bIsPDFA', this.isPDFA);
    xml += this.serializeXmlProp('m_nCsvTxtEncoding', this.csvTxtEncoding);
    xml += this.serializeXmlProp('m_nCsvDelimiter', this.csvDelimiter);
    xml += this.serializeXmlProp('m_nCsvDelimiterChar', this.csvDelimiterChar);
    xml += this.serializeXmlProp('m_bPaid', this.paid);
    xml += this.serializeXmlProp('m_bEmbeddedFonts', this.embeddedFonts);
    xml += this.serializeXmlProp('m_bFromChanges', this.fromChanges);
    xml += this.serializeXmlProp('m_sFontDir', this.fontDir);
    xml += this.serializeXmlProp('m_sThemeDir', this.themeDir);
    if (this.mailMergeSend) {
      xml += this.serializeMailMerge(this.mailMergeSend);
    }
    if (this.thumbnail) {
      xml += this.serializeThumbnail(this.thumbnail);
    }
    if (this.textParams) {
      xml += this.serializeTextParams(this.textParams);
    }
    let jsonParams = this.jsonParams;
    xml += this.serializeXmlProp('m_nLcid', this.lcid);
    xml += this.serializeXmlProp('m_oTimestamp', this.timestamp.toISOString());
    xml += this.serializeXmlProp('m_bIsNoBase64', this.noBase64);
    xml += this.serializeXmlProp('m_sConvertToOrigin', this.convertToOrigin);
    if (this.formatTo === constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF && commonDefines.c_oAscForceSaveTypes.Form === this.forceSaveType) {
      const signingPath = resolveSigningPath(ctx);
      const signingCfg = ctx.getCfg('FileConverter.converter.signing', cfgSigning);
      const {isCloud, certPath} = getCloudSigningMode(signingPath, signingCfg);
      if (certPath && fs.existsSync(certPath)) {
        if (isCloud) {
          xml += this.serializeXmlProp('m_sSigningKeyStorePath', '_placeholder_');
          this._cloudSigningCertPath = certPath;
        } else {
          xml += this.serializeXmlProp('m_sSigningKeyStorePath', certPath);
        }
        jsonParams = mergeSigningMeta(signingCfg, jsonParams);
      }
    }
    xml += this.serializeXmlProp('m_sJsonParams', jsonParams);
    xml += this.serializeLimit(ctx);
    xml += this.serializeOptions(ctx, false, this.oformAsPdf);
    xml += '</TaskQueueDataConvert>';
    fs.writeFileSync(fsPath, xml, {encoding: 'utf8'});
  },
  serializeHidden(ctx) {
    const t = this;
    return co(function* () {
      let xml;
      if (t.password || t.savePassword) {
        xml = '<TaskQueueDataConvert>';
        if (t.password) {
          const password = yield utils.decryptPassword(ctx, t.password);
          xml += t.serializeXmlProp('m_sPassword', password);
        }
        if (t.savePassword) {
          const savePassword = yield utils.decryptPassword(ctx, t.savePassword);
          xml += t.serializeXmlProp('m_sSavePassword', savePassword);
        }
        xml += '</TaskQueueDataConvert>';
      }
      return xml;
    });
  },
  serializeOptions(ctx, isInJwtToken, oformAsPdf) {
    const tenRequesFilteringAgent = ctx.getCfg('services.CoAuthoring.request-filtering-agent', cfgRequesFilteringAgent);
    const tenExternalRequestDirectIfIn = ctx.getCfg('externalRequest.directIfIn', cfgExternalRequestDirectIfIn);
    const tenExternalRequestAction = ctx.getCfg('externalRequest.action', cfgExternalRequestAction);
    const allowList = tenExternalRequestDirectIfIn.allowList;
    let allowNetworkRequest = tenExternalRequestAction.allow;
    let allowPrivateIP = !tenExternalRequestAction.blockPrivateIP && tenRequesFilteringAgent.allowPrivateIPAddress;
    let proxyUrl = tenExternalRequestAction.proxyUrl;
    let proxyUser = tenExternalRequestAction.proxyUser;
    let proxyHeaders = tenExternalRequestAction.proxyHeaders;
    if (allowList.length === 0 && tenExternalRequestDirectIfIn.jwtToken && isInJwtToken) {
      allowNetworkRequest = true;
      allowPrivateIP = true;
      proxyUrl = '';
      proxyUser = null;
      proxyHeaders = {};
    }
    let xml = '';
    xml += '<options>';
    if (allowList.length > 0) {
      xml += this.serializeXmlProp('allowList', allowList.join(';'));
    }
    xml += this.serializeXmlProp('allowNetworkRequest', allowNetworkRequest);
    xml += this.serializeXmlProp('allowPrivateIP', allowPrivateIP);
    if (proxyUrl) {
      xml += this.serializeXmlProp('proxy', proxyUrl);
    }
    if (proxyUser) {
      const user = proxyUser.username;
      const pass = proxyUser.password;
      xml += this.serializeXmlProp('proxyUser', `${user}:${pass}`);
    }
    const proxyHeadersStr = [];
    for (const name in proxyHeaders) {
      proxyHeadersStr.push(`${name}:${proxyHeaders[name]}`);
    }
    if (proxyHeadersStr.length > 0) {
      xml += this.serializeXmlProp('proxyHeader', proxyHeadersStr.join(';'));
    }
    if (undefined !== oformAsPdf) {
      xml += this.serializeXmlProp('oformAsPdf', oformAsPdf);
    }
    xml += '</options>';
    return xml;
  },
  serializeMailMerge(data) {
    let xml = '<m_oMailMergeSend>';
    xml += this.serializeXmlProp('from', data.getFrom());
    xml += this.serializeXmlProp('to', data.getTo());
    xml += this.serializeXmlProp('subject', data.getSubject());
    xml += this.serializeXmlProp('mailFormat', data.getMailFormat());
    xml += this.serializeXmlProp('fileName', data.getFileName());
    xml += this.serializeXmlProp('message', data.getMessage());
    xml += this.serializeXmlProp('recordFrom', data.getRecordFrom());
    xml += this.serializeXmlProp('recordTo', data.getRecordTo());
    xml += this.serializeXmlProp('recordCount', data.getRecordCount());
    xml += this.serializeXmlProp('userid', data.getUserId());
    xml += this.serializeXmlProp('url', data.getUrl());
    xml += '</m_oMailMergeSend>';
    return xml;
  },
  serializeThumbnail(data) {
    let xml = '<m_oThumbnail>';
    xml += this.serializeXmlProp('format', data.getFormat());
    xml += this.serializeXmlProp('aspect', data.getAspect());
    xml += this.serializeXmlProp('first', data.getFirst());
    xml += this.serializeXmlProp('width', data.getWidth());
    xml += this.serializeXmlProp('height', data.getHeight());
    xml += '</m_oThumbnail>';
    return xml;
  },
  serializeTextParams(data) {
    let xml = '<m_oTextParams>';
    xml += this.serializeXmlProp('m_nTextAssociationType', data.getAssociation());
    xml += '</m_oTextParams>';
    return xml;
  },
  serializeLimit(ctx) {
    let xml = '<m_oInputLimits>';
    const tenInputLimits = ctx.getCfg('FileConverter.converter.inputLimits', cfgInputLimits);
    for (let i = 0; i < tenInputLimits.length; ++i) {
      const limit = tenInputLimits[i];
      if (limit.type && limit.zip) {
        xml += '<m_oInputLimit';
        xml += this.serializeXmlAttr('type', limit.type);
        xml += '>';
        xml += '<m_oZip';
        if (limit.zip.compressed) {
          xml += this.serializeXmlAttr('compressed', bytes.parse(limit.zip.compressed));
        }
        if (limit.zip.uncompressed) {
          xml += this.serializeXmlAttr('uncompressed', bytes.parse(limit.zip.uncompressed));
        }
        xml += this.serializeXmlAttr('template', limit.zip.template);
        xml += '/>';
        xml += '</m_oInputLimit>';
      }
    }
    xml += '</m_oInputLimits>';
    return xml;
  },
  serializeXmlProp(name, value) {
    let xml = '';
    //todo check empty and undefined (password?)
    if (null != value) {
      xml += '<' + name + '>';
      xml += utils.encodeXml(value.toString());
      xml += '</' + name + '>';
    } else {
      xml += '<' + name + ' xsi:nil="true" />';
    }
    return xml;
  },
  serializeXmlAttr(name, value) {
    let xml = '';
    if (null != value) {
      xml += ' ' + name + '="';
      xml += utils.encodeXml(value.toString());
      xml += '"';
    }
    return xml;
  }
};

function getTempDir() {
  const tempDir = os.tmpdir();
  const now = new Date();
  let newTemp;
  while (!newTemp || fs.existsSync(newTemp)) {
    const newName = [TEMP_PREFIX, now.getFullYear(), now.getMonth(), now.getDate(), '-', (Math.random() * 0x100000000 + 1).toString(36)].join('');
    newTemp = path.join(tempDir, newName);
  }
  fs.mkdirSync(newTemp);
  const sourceDir = path.join(newTemp, 'source');
  fs.mkdirSync(sourceDir);
  const resultDir = path.join(newTemp, 'result');
  fs.mkdirSync(resultDir);
  return {temp: newTemp, source: sourceDir, result: resultDir};
}
function* isUselessConvertion(ctx, task, cmd) {
  if (task.getFromChanges() && 'sfc' === cmd.getCommand()) {
    const selectRes = yield taskResult.select(ctx, cmd.getDocId());
    const row = selectRes.length > 0 ? selectRes[0] : null;
    if (utils.isUselesSfc(row, cmd)) {
      ctx.logger.warn('isUselessConvertion return true. row=%j', row);
      return constants.CONVERT_PARAMS;
    }
  }
  return constants.NO_ERROR;
}
async function changeFormatToExtendedPdf(ctx, dataConvert, cmd) {
  const originFormat = cmd.getOriginFormat();
  const isOriginFormatWithForms =
    constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === originFormat ||
    constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM === originFormat ||
    constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCXF === originFormat;
  const isFormatToPdf =
    constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF === dataConvert.formatTo ||
    constants.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDFA === dataConvert.formatTo;
  if (isFormatToPdf && isOriginFormatWithForms) {
    const format = await formatChecker.getDocumentFormatByFile(dataConvert.fileFrom);
    if (constants.AVS_OFFICESTUDIO_FILE_CANVAS_WORD === format) {
      ctx.logger.debug('change format to extended pdf');
      dataConvert.formatTo = constants.AVS_OFFICESTUDIO_FILE_DOCUMENT_OFORM_PDF;
    }
  }
}
/**
 * Replace an empty file with a localized template of the requested format, if available.
 * @param {operationContext} ctx - Operation context for config and logging
 * @param {string} fileFrom - Path to the file to check/replace
 * @param {string} ext - Target extension (e.g., 'docx', 'xlsx', 'pptx')
 * @param {number} _lcid - Optional locale identifier to select template locale
 */
function replaceEmptyFile(ctx, fileFrom, ext, _lcid) {
  const tenNewFileTemplate = ctx.getCfg('services.CoAuthoring.server.newFileTemplate', cfgNewFileTemplate);
  if (!fs.existsSync(fileFrom) || 0 === fs.lstatSync(fileFrom).size) {
    let locale = constants.TEMPLATES_DEFAULT_LOCALE;
    if (_lcid) {
      let localeNew = lcid.from(_lcid);
      if (localeNew) {
        localeNew = localeNew.replace(/_/g, '-');
        if (fs.existsSync(path.join(tenNewFileTemplate, localeNew))) {
          locale = localeNew;
        } else {
          ctx.logger.debug('replaceEmptyFile empty locale dir locale=%s', localeNew);
        }
      }
    }
    const fileTemplatePath = path.join(tenNewFileTemplate, locale, 'new.');
    if (fs.existsSync(fileTemplatePath + ext)) {
      ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale);
      fs.copyFileSync(fileTemplatePath + ext, fileFrom);
    } else {
      const format = formatChecker.getFormatFromString(ext);
      let editorFormat;
      if (formatChecker.isDocumentFormat(format)) {
        editorFormat = 'docx';
      } else if (formatChecker.isSpreadsheetFormat(format)) {
        editorFormat = 'xlsx';
      } else if (formatChecker.isPresentationFormat(format)) {
        editorFormat = 'pptx';
      }
      if (fs.existsSync(fileTemplatePath + editorFormat)) {
        ctx.logger.debug('replaceEmptyFile format=%s locale=%s', ext, locale);
        fs.copyFileSync(fileTemplatePath + editorFormat, fileFrom);
      }
    }
  }
}
function* downloadFile(ctx, uri, fileFrom, withAuthorization, isInJwtToken, opt_headers) {
  const tenMaxDownloadBytes = ctx.getCfg('FileConverter.converter.maxDownloadBytes', cfgMaxDownloadBytes);
  const tenDownloadTimeout = ctx.getCfg('FileConverter.converter.downloadTimeout', cfgDownloadTimeout);
  const tenDownloadAttemptMaxCount = ctx.getCfg('FileConverter.converter.downloadAttemptMaxCount', cfgDownloadAttemptMaxCount);
  const tenDownloadAttemptDelay = ctx.getCfg('FileConverter.converter.downloadAttemptDelay', cfgDownloadAttemptDelay);
  let res = constants.CONVERT_DOWNLOAD;
  let data = null;
  let sha256 = null;
  let downloadAttemptCount = 0;
  const urlParsed = url.parse(uri);
  const filterStatus = yield* utils.checkHostFilter(ctx, urlParsed.hostname);
  if (0 == filterStatus) {
    while (constants.NO_ERROR !== res && downloadAttemptCount++ < tenDownloadAttemptMaxCount) {
      try {
        let authorization;
        if (utils.canIncludeOutboxAuthorization(ctx, uri) && withAuthorization) {
          const secret = yield tenantManager.getTenantSecret(ctx, commonDefines.c_oAscSecretType.Outbox);
          authorization = utils.fillJwtForRequest(ctx, {url: uri}, secret, false);
        }
        const getRes = yield utils.downloadUrlPromise(ctx, uri, tenDownloadTimeout, tenMaxDownloadBytes, authorization, isInJwtToken, opt_headers);
        data = getRes.body;
        sha256 = getRes.sha256;
        res = constants.NO_ERROR;
      } catch (err) {
        res = constants.CONVERT_DOWNLOAD;
        ctx.logger.error('error downloadFile:url=%s;attempt=%d;code:%s;connect:%s %s', uri, downloadAttemptCount, err.code, err.connect, err.stack);
        //not continue attempts if timeout
        if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') {
          break;
        } else if (err.code === 'EMSGSIZE') {
          res = constants.CONVERT_LIMITS;
          break;
        } else {
          yield utils.sleep(tenDownloadAttemptDelay);
        }
      }
    }
    if (constants.NO_ERROR === res) {
      ctx.logger.debug('downloadFile complete filesize=%d sha256=%s', data.length, sha256);
      fs.writeFileSync(fileFrom, data);
    }
  } else {
    ctx.logger.error('checkIpFilter error:url=%s;code:%s;', uri, filterStatus);
    res = constants.CONVERT_DOWNLOAD;
  }
  return res;
}
function* downloadFileFromStorage(ctx, strPath, dir, opt_specialDir) {
  const list = yield storage.listObjects(ctx, strPath, opt_specialDir);
  ctx.logger.debug('downloadFileFromStorage list %s', list.toString());
  //create dirs
  const dirsToCreate = [];
  const dirStruct = {};
  list.forEach(file => {
    let curDirPath = dir;
    const curDirStruct = dirStruct;
    const parts = storage.getRelativePath(strPath, file).split('/');
    for (let i = 0; i < parts.length - 1; ++i) {
      const part = parts[i];
      curDirPath = path.join(curDirPath, part);
      if (!curDirStruct[part]) {
        curDirStruct[part] = {};
        dirsToCreate.push(curDirPath);
      }
    }
  });
  //make dirs
  for (let i = 0; i < dirsToCreate.length; ++i) {
    fs.mkdirSync(dirsToCreate[i]);
  }
  //download
  //todo Promise.all
  for (let i = 0; i < list.length; ++i) {
    const file = list[i];
    const fileRel = storage.getRelativePath(strPath, file);
    const data = yield storage.getObject(ctx, file, opt_specialDir);
    fs.writeFileSync(path.join(dir, fileRel), data);
  }
  return list.length;
}
function* processDownloadFromStorage(ctx, dataConvert, cmd, task, tempDirs, authorProps) {
  const tenEditor = ctx.getCfg('services.CoAuthoring.editor', cfgEditor);
  let res = constants.NO_ERROR;
  let concatDir;
  let concatTemplate;
  if (task.getFromOrigin() || task.getFromSettings()) {
    if (task.getFromChanges()) {
      const changesDir = path.join(tempDirs.source, constants.CHANGES_NAME);
      fs.mkdirSync(changesDir);
      let filesCount = 0;
      if (cmd.getSaveKey()) {
        filesCount = yield* downloadFileFromStorage(ctx, cmd.getDocId() + cmd.getSaveKey(), changesDir);
      }
      if (filesCount > 0) {
        concatDir = changesDir;
        concatTemplate = 'changes0';
      } else {
        dataConvert.fromChanges = false;
        task.setFromChanges(dataConvert.fromChanges);
      }
    }
    dataConvert.fileFrom = path.join(tempDirs.source, 'origin.' + cmd.getFormat());
  } else {
    //overwrite some files from m_sKey (for example Editor.bin or changes)
    if (cmd.getSaveKey()) {
      yield* downloadFileFromStorage(ctx, cmd.getDocId() + cmd.getSaveKey(), tempDirs.source);
    }
    const format = cmd.getFormat() || 'bin';
    dataConvert.fileFrom = path.join(tempDirs.source, 'Editor.' + format);
    concatDir = tempDirs.source;
  }
  if (!utils.checkPathTraversal(ctx, dataConvert.key, tempDirs.source, dataConvert.fileFrom)) {
    return constants.CONVERT_PARAMS;
  }
  //mail merge
  const mailMergeSend = cmd.getMailMergeSend();
  if (mailMergeSend) {
    yield* downloadFileFromStorage(ctx, cmd.getDocId() + mailMergeSend.getJsonKey(), tempDirs.source);
    concatDir = tempDirs.source;
  }
  if (concatDir) {
    yield* concatFiles(concatDir, concatTemplate);
    if (concatTemplate) {
      const filenames = fs.readdirSync(concatDir);
      filenames.forEach(file => {
        if (file.match(new RegExp(`${concatTemplate}\\d+\\.`))) {
          fs.rmSync(path.join(concatDir, file));
        }
      });
    }
  }
  //todo rework
  if (!fs.existsSync(dataConvert.fileFrom)) {
    if (fs.existsSync(path.join(tempDirs.source, 'origin.docx'))) {
      dataConvert.fileFrom = path.join(tempDirs.source, 'origin.docx');
    } else if (fs.existsSync(path.join(tempDirs.source, 'origin.xlsx'))) {
      dataConvert.fileFrom = path.join(tempDirs.source, 'origin.xlsx');
    } else if (fs.existsSync(path.join(tempDirs.source, 'origin.pptx'))) {
      dataConvert.fileFrom = path.join(tempDirs.source, 'origin.pptx');
    } else if (fs.existsSync(path.join(tempDirs.source, 'origin.pdf'))) {
      dataConvert.fileFrom = path.join(tempDirs.source, 'origin.pdf');
    }
    if (fs.existsSync(dataConvert.fileFrom)) {
      const fileFromNew = path.join(path.dirname(dataConvert.fileFrom), 'Editor.bin');
      fs.renameSync(dataConvert.fileFrom, fileFromNew);
      dataConvert.fileFrom = fileFromNew;
    }
  }

  yield changeFormatToExtendedPdf(ctx, dataConvert, cmd);

  if (task.getFromChanges() && !(task.getFromOrigin() || task.getFromSettings())) {
    const sha256 = yield utils.checksumFile('sha256', dataConvert.fileFrom);
    if (tenEditor['binaryChanges']) {
      res = yield* processChangesBin(ctx, tempDirs, task, cmd, authorProps, sha256);
    } else {
      res = yield* processChangesBase64(ctx, tempDirs, task, cmd, authorProps, sha256);
    }
  }
  return res;
}

function* concatFiles(source, template) {
  template = template || 'Editor';
  //concatenate EditorN.ext parts in Editor.ext
  const list = yield utils.listObjects(source, true);
  list.sort(utils.compareStringByLength);
  const createdTargets = new Set();
  for (let i = 0; i < list.length; ++i) {
    const file = list[i];
    if (file.match(new RegExp(`${template}\\d+\\.`))) {
      const target = file.replace(new RegExp(`(${template})\\d+(\\..*)`), '$1$2');
      const isFirst = !createdTargets.has(target);
      const writeOpts = isFirst ? undefined : {flags: 'a'};
      const writeStream = yield utils.promiseCreateWriteStream(target, writeOpts);
      const readStream = yield utils.promiseCreateReadStream(file);
      // Use raw pipeline for file-to-file operations to surface real errors
      yield pipeline(readStream, writeStream);
      if (isFirst) {
        createdTargets.add(target);
      }
    }
  }
}
function* processChangesBin(ctx, tempDirs, task, cmd, authorProps, sha256) {
  const tenStreamWriterBufferSize = ctx.getCfg('FileConverter.converter.streamWriterBufferSize', cfgStreamWriterBufferSize);
  const tenMaxRequestChanges = ctx.getCfg('services.CoAuthoring.server.maxRequestChanges', cfgMaxRequestChanges);
  let res = constants.NO_ERROR;
  const changesDir = path.join(tempDirs.source, constants.CHANGES_NAME);
  fs.mkdirSync(changesDir);
  let indexFile = 0;
  let changesAuthor = null;
  let changesAuthorUnique = null;
  let changesIndex = null;
  const changesHistory = {
    serverVersion: commonDefines.buildVersion,
    changes: []
  };
  const forceSave = cmd.getForceSave();
  let forceSaveTime;
  let forceSaveIndex = Number.MAX_VALUE;
  if (forceSave && undefined !== forceSave.getTime() && undefined !== forceSave.getIndex()) {
    forceSaveTime = forceSave.getTime();
    forceSaveIndex = forceSave.getIndex();
  }
  const extChangeInfo = cmd.getExternalChangeInfo();
  let extChanges;
  if (extChangeInfo) {
    extChanges = [
      {
        id: cmd.getDocId(),
        change_id: 0,
        change_data: Buffer.alloc(0),
        user_id: extChangeInfo.user_id,
        user_id_original: extChangeInfo.user_id_original,
        user_name: extChangeInfo.user_name,
        change_date: new Date(extChangeInfo.change_date)
      }
    ];
  }

  let streamObj = yield* streamCreateBin(ctx, changesDir, indexFile++, {highWaterMark: tenStreamWriterBufferSize});
  yield* streamWriteBin(streamObj, Buffer.from(utils.getChangesFileHeader(), 'utf-8'));
  let curIndexStart = 0;
  let curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex);
  while (curIndexStart < curIndexEnd || extChanges) {
    let changes = [];
    if (curIndexStart < curIndexEnd) {
      changes = yield baseConnector.getChangesPromise(ctx, cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime);
      if (changes.length > 0 && changes[0].change_data.subarray(0, 'ENCRYPTED;'.length).includes('ENCRYPTED;')) {
        ctx.logger.warn('processChanges encrypted changes');
        //todo sql request instead?
        res = constants.EDITOR_CHANGES;
      }
      res = yield* isUselessConvertion(ctx, task, cmd);
      if (constants.NO_ERROR !== res) {
        break;
      }
    }
    if (0 === changes.length && extChanges) {
      changes = extChanges;
    }
    extChanges = undefined;
    for (let i = 0; i < changes.length; ++i) {
      const change = changes[i];
      if (null === changesAuthor || changesAuthor !== change.user_id_original) {
        if (null !== changesAuthor) {
          yield* streamEndBin(streamObj);
          streamObj = yield* streamCreateBin(ctx, changesDir, indexFile++);
          yield* streamWriteBin(streamObj, Buffer.from(utils.getChangesFileHeader(), 'utf-8'));
        }
        const strDate = baseConnector.getDateTime(change.change_date);
        changesHistory.changes.push({documentSha256: sha256, created: strDate, user: {id: change.user_id_original, name: change.user_name}});
      }
      changesAuthor = change.user_id_original;
      changesAuthorUnique = change.user_id;
      yield* streamWriteBin(streamObj, change.change_data);
      streamObj.isNoChangesInFile = false;
    }
    if (changes.length > 0) {
      authorProps.lastModifiedBy = changes[changes.length - 1].user_name;
      authorProps.modified = changes[changes.length - 1].change_date.toISOString().slice(0, 19) + 'Z';
    }
    if (changes.length === curIndexEnd - curIndexStart) {
      curIndexStart += tenMaxRequestChanges;
      curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex);
    } else {
      break;
    }
  }
  yield* streamEndBin(streamObj);
  if (streamObj.isNoChangesInFile) {
    fs.unlinkSync(streamObj.filePath);
  }
  if (null !== changesAuthorUnique) {
    changesIndex = utils.getIndexFromUserId(changesAuthorUnique, changesAuthor);
  }
  if (
    null == changesAuthor &&
    null == changesIndex &&
    forceSave &&
    undefined !== forceSave.getAuthorUserId() &&
    undefined !== forceSave.getAuthorUserIndex()
  ) {
    changesAuthor = forceSave.getAuthorUserId();
    changesIndex = forceSave.getAuthorUserIndex();
  }
  cmd.setUserId(changesAuthor);
  cmd.setUserIndex(changesIndex);
  fs.writeFileSync(path.join(tempDirs.result, 'changesHistory.json'), JSON.stringify(changesHistory), 'utf8');
  ctx.logger.debug('processChanges end');
  return res;
}

function* streamCreateBin(ctx, changesDir, indexFile, opt_options) {
  const fileName = constants.CHANGES_NAME + indexFile + '.bin';
  const filePath = path.join(changesDir, fileName);
  const writeStream = yield utils.promiseCreateWriteStream(filePath, opt_options);
  writeStream.on('error', err => {
    //todo integrate error handle in main thread (probable: set flag here and check it in main thread)
    ctx.logger.error('WriteStreamError %s', err.stack);
  });
  return {writeStream, filePath, isNoChangesInFile: true};
}

function* streamWriteBin(streamObj, buf) {
  if (!streamObj.writeStream.write(buf)) {
    yield utils.promiseWaitDrain(streamObj.writeStream);
  }
}

function* streamEndBin(streamObj) {
  streamObj.writeStream.end();
  yield utils.promiseWaitClose(streamObj.writeStream);
}
function* processChangesBase64(ctx, tempDirs, task, cmd, authorProps, sha256) {
  const tenStreamWriterBufferSize = ctx.getCfg('FileConverter.converter.streamWriterBufferSize', cfgStreamWriterBufferSize);
  const tenMaxRequestChanges = ctx.getCfg('services.CoAuthoring.server.maxRequestChanges', cfgMaxRequestChanges);
  let res = constants.NO_ERROR;
  const changesDir = path.join(tempDirs.source, constants.CHANGES_NAME);
  fs.mkdirSync(changesDir);
  let indexFile = 0;
  let changesAuthor = null;
  let changesAuthorUnique = null;
  let changesIndex = null;
  const changesHistory = {
    serverVersion: commonDefines.buildVersion,
    changes: []
  };
  const forceSave = cmd.getForceSave();
  let forceSaveTime;
  let forceSaveIndex = Number.MAX_VALUE;
  if (forceSave && undefined !== forceSave.getTime() && undefined !== forceSave.getIndex()) {
    forceSaveTime = forceSave.getTime();
    forceSaveIndex = forceSave.getIndex();
  }
  const extChangeInfo = cmd.getExternalChangeInfo();
  let extChanges;
  if (extChangeInfo) {
    extChanges = [
      {
        id: cmd.getDocId(),
        change_id: 0,
        change_data: '',
        user_id: extChangeInfo.user_id,
        user_id_original: extChangeInfo.user_id_original,
        user_name: extChangeInfo.user_name,
        change_date: new Date(extChangeInfo.change_date)
      }
    ];
  }

  let streamObj = yield* streamCreate(ctx, changesDir, indexFile++, {highWaterMark: tenStreamWriterBufferSize});
  let curIndexStart = 0;
  let curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex);
  while (curIndexStart < curIndexEnd || extChanges) {
    let changes = [];
    if (curIndexStart < curIndexEnd) {
      changes = yield baseConnector.getChangesPromise(ctx, cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime);
      if (changes.length > 0 && changes[0].change_data.startsWith('ENCRYPTED;')) {
        ctx.logger.warn('processChanges encrypted changes');
        //todo sql request instead?
        res = constants.EDITOR_CHANGES;
      }
      res = yield* isUselessConvertion(ctx, task, cmd);
      if (constants.NO_ERROR !== res) {
        break;
      }
    }
    if (0 === changes.length && extChanges) {
      changes = extChanges;
    }
    extChanges = undefined;
    for (let i = 0; i < changes.length; ++i) {
      const change = changes[i];
      if (null === changesAuthor || changesAuthor !== change.user_id_original) {
        if (null !== changesAuthor) {
          yield* streamEnd(streamObj, ']');
          streamObj = yield* streamCreate(ctx, changesDir, indexFile++);
        }
        const strDate = baseConnector.getDateTime(change.change_date);
        changesHistory.changes.push({documentSha256: sha256, created: strDate, user: {id: change.user_id_original, name: change.user_name}});
        yield* streamWrite(streamObj, '[');
      } else {
        yield* streamWrite(streamObj, ',');
      }
      changesAuthor = change.user_id_original;
      changesAuthorUnique = change.user_id;
      yield* streamWrite(streamObj, change.change_data);
      streamObj.isNoChangesInFile = false;
    }
    if (changes.length > 0) {
      authorProps.lastModifiedBy = changes[changes.length - 1].user_name;
      authorProps.modified = changes[changes.length - 1].change_date.toISOString().slice(0, 19) + 'Z';
    }
    if (changes.length === curIndexEnd - curIndexStart) {
      curIndexStart += tenMaxRequestChanges;
      curIndexEnd = Math.min(curIndexStart + tenMaxRequestChanges, forceSaveIndex);
    } else {
      break;
    }
  }
  yield* streamEnd(streamObj, ']');
  if (streamObj.isNoChangesInFile) {
    fs.unlinkSync(streamObj.filePath);
  }
  if (null !== changesAuthorUnique) {
    changesIndex = utils.getIndexFromUserId(changesAuthorUnique, changesAuthor);
  }
  if (
    null == changesAuthor &&
    null == changesIndex &&
    forceSave &&
    undefined !== forceSave.getAuthorUserId() &&
    undefined !== forceSave.getAuthorUserIndex()
  ) {
    changesAuthor = forceSave.getAuthorUserId();
    changesIndex = forceSave.getAuthorUserIndex();
  }
  cmd.setUserId(changesAuthor);
  cmd.setUserIndex(changesIndex);
  fs.writeFileSync(path.join(tempDirs.result, 'changesHistory.json'), JSON.stringify(changesHistory), 'utf8');
  ctx.logger.debug('processChanges end');
  return res;
}

function* streamCreate(ctx, changesDir, indexFile, opt_options) {
  const fileName = constants.CHANGES_NAME + indexFile + '.json';
  const filePath = path.join(changesDir, fileName);
  const writeStream = yield utils.promiseCreateWriteStream(filePath, opt_options);
  writeStream.on('error', err => {
    //todo integrate error handle in main thread (probable: set flag here and check it in main thread)
    ctx.logger.error('WriteStreamError %s', err.stack);
  });
  return {writeStream, filePath, isNoChangesInFile: true};
}

function* streamWrite(streamObj, text) {
  if (!streamObj.writeStream.write(text, 'utf8')) {
    yield utils.promiseWaitDrain(streamObj.writeStream);
  }
}

function* streamEnd(streamObj, text) {
  streamObj.writeStream.end(text, 'utf8');
  yield utils.promiseWaitClose(streamObj.writeStream);
}
function* processUploadToStorage(ctx, dir, storagePath, calcChecksum, opt_specialDirDst, opt_ignorPrefix) {
  let list = yield utils.listObjects(dir);
  if (opt_ignorPrefix) {
    list = list.filter(dir => !dir.startsWith(opt_ignorPrefix));
  }
  if (list.length < MAX_OPEN_FILES) {
    yield* processUploadToStorageChunk(ctx, list, dir, storagePath, calcChecksum, opt_specialDirDst);
  } else {
    for (let i = 0, j = list.length; i < j; i += MAX_OPEN_FILES) {
      yield* processUploadToStorageChunk(ctx, list.slice(i, i + MAX_OPEN_FILES), dir, storagePath, calcChecksum, opt_specialDirDst);
    }
  }
}
function* processUploadToStorageChunk(ctx, list, dir, storagePath, calcChecksum, opt_specialDirDst) {
  const promises = list.reduce((r, curValue) => {
    const localValue = storagePath + '/' + curValue.substring(dir.length + 1);
    let checksum;
    if (calcChecksum) {
      checksum = utils.checksumFile('sha256', curValue).then(result => {
        ctx.logger.debug('processUploadToStorageChunk path=%s; sha256=%s', localValue, result);
      });
    }
    const upload = storage.uploadObject(ctx, localValue, curValue, opt_specialDirDst);
    r.push(checksum, upload);
    return r;
  }, []);
  yield Promise.all(promises);
}
function* processUploadToStorageErrorFile(ctx, dataConvert, tempDirs, childRes, exitCode, exitSignal, error) {
  const tenErrorFiles = ctx.getCfg('FileConverter.converter.errorfiles', cfgErrorFiles);
  if (!tenErrorFiles) {
    return;
  }
  let output = '';
  if (undefined !== childRes.stdout) {
    output += `stdout:${childRes.stdout}\n`;
  }
  if (undefined !== childRes.stderr) {
    output += `stderr:${childRes.stderr}\n`;
  }
  output += `ExitCode (code=${exitCode};signal=${exitSignal};error:${error})`;
  const outputPath = path.join(tempDirs.temp, 'console.txt');
  fs.writeFileSync(outputPath, output, {encoding: 'utf8'});

  //ignore result dir with temp dir inside(see m_sTempDir param) to reduce the amount of data transferred
  const ignorePrefix = path.normalize(tempDirs.result);
  const format = path.extname(dataConvert.fileFrom).substring(1) || 'unknown';

  yield* processUploadToStorage(ctx, tempDirs.temp, format + '/' + dataConvert.key, false, tenErrorFiles, ignorePrefix);
  ctx.logger.debug('processUploadToStorage error complete(id=%s)', dataConvert.key);
}
function writeProcessOutputToLog(ctx, childRes, isDebug) {
  if (childRes) {
    if (undefined !== childRes.stdout) {
      if (isDebug) {
        ctx.logger.debug('stdout:%s', childRes.stdout);
      } else {
        ctx.logger.error('stdout:%s', childRes.stdout);
      }
    }
    if (undefined !== childRes.stderr) {
      if (isDebug) {
        ctx.logger.debug('stderr:%s', childRes.stderr);
      } else {
        ctx.logger.error('stderr:%s', childRes.stderr);
      }
    }
  }
}
function* postProcess(ctx, cmd, dataConvert, tempDirs, childRes, error, isTimeout) {
  let exitCode = 0;
  let exitSignal = null;
  if (childRes) {
    exitCode = childRes.status;
    exitSignal = childRes.signal;
  }
  //CONVERT_CELLLIMITS is not an error, but an indicator that data was lost during opening (can be displayed as an error)
  if ((0 !== exitCode && constants.CONVERT_CELLLIMITS !== -exitCode) || null !== exitSignal) {
    if (-1 !== exitCodesReturn.indexOf(-exitCode)) {
      error = -exitCode;
    } else if (isTimeout) {
      error = constants.CONVERT_TIMEOUT;
    } else {
      error = constants.CONVERT;
    }
    if (-1 !== exitCodesMinorError.indexOf(error)) {
      writeProcessOutputToLog(ctx, childRes, true);
      ctx.logger.debug('ExitCode (code=%d;signal=%s;error:%d)', exitCode, exitSignal, error);
    } else {
      writeProcessOutputToLog(ctx, childRes, false);
      ctx.logger.error('ExitCode (code=%d;signal=%s;error:%d)', exitCode, exitSignal, error);
      yield* processUploadToStorageErrorFile(ctx, dataConvert, tempDirs, childRes, exitCode, exitSignal, error);
    }
  } else {
    writeProcessOutputToLog(ctx, childRes, true);
    ctx.logger.debug('ExitCode (code=%d;signal=%s;error:%d)', exitCode, exitSignal, error);
  }
  let existFile = false;
  try {
    existFile = fs.lstatSync(dataConvert.fileTo).isFile();
  } catch (_err) {
    existFile = false;
  }
  if (!existFile && dataConvert.fileTo) {
    //todo review. the stub in the case of AVS_OFFICESTUDIO_FILE_OTHER_OOXML x2t changes the file extension.
    const fileToBasename = path.basename(dataConvert.fileTo, path.extname(dataConvert.fileTo));
    const fileToDir = path.dirname(dataConvert.fileTo);
    const files = fs.readdirSync(fileToDir);
    for (let i = 0; i < files.length; ++i) {
      const fileCur = files[i];
      if (0 == fileCur.indexOf(fileToBasename)) {
        dataConvert.fileTo = path.join(fileToDir, fileCur);
        existFile = true;
        break;
      }
    }
    // todo: breaks open-pdf (no output file is valid); uncomment when fixed
    // if (constants.NO_ERROR === error && !existFile) {
    //   //return CONVERT error so canvasservice treats this as a failed conversion,
    //   //not as missing cache (empty storage), which would trigger an infinite retry loop
    //   ctx.logger.error('Conversion produced no output file');
    //   error = constants.CONVERT;
    // }
  }
  if (-1 !== exitCodesUpload.indexOf(error)) {
    if (-1 !== exitCodesCopyOrigin.indexOf(error)) {
      const originPath = path.join(path.dirname(dataConvert.fileTo), 'origin' + path.extname(dataConvert.fileFrom));
      if (!fs.existsSync(dataConvert.fileTo)) {
        fs.copyFileSync(dataConvert.fileFrom, originPath);
        ctx.logger.debug('copyOrigin complete');
      }
    }
    // Cloud signing: replace x2t placeholder with real signature (KMS/CloudHSM/CSC) before upload
    if (dataConvert._cloudSigningCertPath) {
      try {
        const signingCfg = ctx.getCfg('FileConverter.converter.signing', cfgSigning);
        yield performCloudSigning(ctx, dataConvert.fileTo, dataConvert._cloudSigningCertPath, signingCfg);
        ctx.logger.debug('Cloud signing complete');
      } catch (cloudErr) {
        ctx.logger.error('Cloud signing failed: %s', cloudErr.stack);
      }
    }
    //todo clarify calcChecksum conditions
    const calcChecksum = 0 === (constants.AVS_OFFICESTUDIO_FILE_CANVAS & cmd.getOutputFormat());
    yield* processUploadToStorage(ctx, tempDirs.result, dataConvert.key, calcChecksum);
    ctx.logger.debug('processUploadToStorage complete');
  }
  cmd.setStatusInfo(error);
  cmd.setOutputPath(path.basename(dataConvert.fileTo));
  if (!cmd.getTitle()) {
    cmd.setTitle(cmd.getOutputPath());
  }

  const queueData = new commonDefines.TaskQueueData();
  queueData.setCtx(ctx);
  queueData.setCmd(cmd);
  ctx.logger.debug('output (data=%j)', queueData);
  return queueData;
}

function* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken) {
  const tenX2tPath = ctx.getCfg('FileConverter.converter.x2tPath', cfgX2tPath);
  const tenDocbuilderPath = ctx.getCfg('FileConverter.converter.docbuilderPath', cfgDocbuilderPath);
  const tenArgs = ctx.getCfg('FileConverter.converter.args', cfgArgs);
  let childRes,
    isTimeout = false;
  let childArgs;
  if (tenArgs.length > 0) {
    childArgs = tenArgs.trim().replace(/  +/g, ' ').split(' ');
  } else {
    childArgs = [];
  }
  let processPath;
  if (!builderParams) {
    processPath = tenX2tPath;
    const paramsFile = path.join(tempDirs.temp, 'params.xml');
    dataConvert.serialize(ctx, paramsFile);
    childArgs.push(paramsFile);
    const hiddenXml = yield dataConvert.serializeHidden(ctx);
    if (hiddenXml) {
      childArgs.push(hiddenXml);
    }
  } else {
    fs.mkdirSync(path.join(tempDirs.result, 'output'));
    processPath = tenDocbuilderPath;
    childArgs.push('--check-fonts=0');
    childArgs.push('--save-use-only-names=' + tempDirs.result + '/output');
    if (builderParams.argument) {
      childArgs.push(`--argument=${JSON.stringify(builderParams.argument)}`);
    }
    childArgs.push('--options=' + dataConvert.serializeOptions(ctx, isInJwtToken));
    childArgs.push(dataConvert.fileFrom);
  }
  let timeoutId;
  try {
    const tenSpawnOptions = ctx.getCfg('FileConverter.converter.spawnOptions', cfgSpawnOptions);
    //copy to avoid modification of global cfgSpawnOptions
    const spawnOptions = Object.assign({}, tenSpawnOptions);
    spawnOptions.env = Object.assign({}, process.env, spawnOptions.env);
    if (authorProps.lastModifiedBy && authorProps.modified) {
      spawnOptions.env['LAST_MODIFIED_BY'] = authorProps.lastModifiedBy;
      spawnOptions.env['MODIFIED'] = authorProps.modified;
    }
    const spawnAsyncPromise = spawnAsync(processPath, childArgs, spawnOptions);
    childRes = spawnAsyncPromise.child;
    const waitMS = Math.max(0, task.getVisibilityTimeout() * 1000 - (new Date().getTime() - getTaskTime.getTime()));
    timeoutId = setTimeout(() => {
      isTimeout = true;
      timeoutId = undefined;
      //close stdio streams to enable emit 'close' event even if HtmlFileInternal is hung-up
      childRes.stdin.end();
      childRes.stdout.destroy();
      childRes.stderr.destroy();
      childRes.kill();
    }, waitMS);
    childRes = yield spawnAsyncPromise;
  } catch (err) {
    if (null === err.status) {
      ctx.logger.error('error spawnAsync %s', err.stack);
    } else {
      ctx.logger.debug('error spawnAsync %s', err.stack);
    }
    childRes = err;
  }
  if (undefined !== timeoutId) {
    clearTimeout(timeoutId);
  }
  return {childRes, isTimeout};
}

function* ExecuteTask(ctx, task) {
  const tenForgottenFiles = ctx.getCfg('services.CoAuthoring.server.forgottenfiles', cfgForgottenFiles);
  const tenForgottenFilesName = ctx.getCfg('services.CoAuthoring.server.forgottenfilesname', cfgForgottenFilesName);
  let startDate = null;
  let curDate = null;
  if (clientStatsD) {
    startDate = curDate = new Date();
  }
  const getTaskTime = new Date();
  const cmd = task.getCmd();
  const dataConvert = new TaskQueueDataConvert(ctx, task);
  ctx.logger.info('Start Task');
  let error = constants.NO_ERROR;
  const tempDirs = getTempDir();
  const fileTo = task.getToFile();
  dataConvert.fileTo = fileTo ? path.join(tempDirs.result, fileTo) : '';
  const builderParams = cmd.getBuilderParams();
  const authorProps = {lastModifiedBy: null, modified: null};
  let isInJwtToken = cmd.getWithAuthorization();
  error = yield* isUselessConvertion(ctx, task, cmd);
  if (constants.NO_ERROR !== error) {
    ctx.logger.error('constants.NO_ERROR !== error', error);
  } else if (cmd.getUrl()) {
    const format = cmd.getFormat();
    dataConvert.fileFrom = path.join(tempDirs.source, dataConvert.key + '.' + format);
    if (utils.checkPathTraversal(ctx, dataConvert.key, tempDirs.source, dataConvert.fileFrom)) {
      let url = cmd.getUrl();
      let withAuthorization = cmd.getWithAuthorization();
      let headers;
      let fileSize;
      const wopiParams = cmd.getWopiParams();
      if (wopiParams) {
        withAuthorization = false;
        isInJwtToken = true;
        const fileInfo = wopiParams.commonInfo?.fileInfo;
        fileSize = fileInfo?.Size;
        ({url, headers} = yield wopiUtils.getWopiFileUrl(ctx, fileInfo, wopiParams.userAuth));
      }
      if (undefined === fileSize || fileSize > 0) {
        error = yield* downloadFile(ctx, url, dataConvert.fileFrom, withAuthorization, isInJwtToken, headers);
      }
      if (constants.NO_ERROR === error) {
        replaceEmptyFile(ctx, dataConvert.fileFrom, format, cmd.getLCID());
      }
      if (clientStatsD) {
        clientStatsD.timing('conv.downloadFile', new Date() - curDate);
        curDate = new Date();
      }
    } else {
      error = constants.CONVERT_PARAMS;
    }
  } else if (cmd.getSaveKey() || task.getFromOrigin() || task.getFromSettings()) {
    yield* downloadFileFromStorage(ctx, cmd.getDocId(), tempDirs.source);
    ctx.logger.debug('downloadFileFromStorage complete');
    if (clientStatsD) {
      clientStatsD.timing('conv.downloadFileFromStorage', new Date() - curDate);
      curDate = new Date();
    }
    error = yield* processDownloadFromStorage(ctx, dataConvert, cmd, task, tempDirs, authorProps);
  } else if (cmd.getForgotten()) {
    yield* downloadFileFromStorage(ctx, cmd.getForgotten(), tempDirs.source, tenForgottenFiles);
    ctx.logger.debug('downloadFileFromStorage complete');
    const list = yield utils.listObjects(tempDirs.source, false);
    if (list.length > 0) {
      dataConvert.fileFrom = list[0];
      //store indicator file to determine if opening was from the forgotten file
      const forgottenMarkPath = tempDirs.result + '/' + tenForgottenFilesName + '.txt';
      fs.writeFileSync(forgottenMarkPath, tenForgottenFilesName, {encoding: 'utf8'});
    } else {
      error = constants.UNKNOWN;
    }
  } else if (builderParams) {
    //in cause script in POST body
    yield* downloadFileFromStorage(ctx, cmd.getDocId(), tempDirs.source);
    ctx.logger.debug('downloadFileFromStorage complete');
    const list = yield utils.listObjects(tempDirs.source, false);
    if (list.length > 0) {
      dataConvert.fileFrom = list[0];
    }
  } else {
    error = constants.UNKNOWN;
  }
  let childRes = null;
  let isTimeout = false;
  if (constants.NO_ERROR === error) {
    ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken));
    const canRollback =
      childRes &&
      0 !== childRes.status &&
      !isTimeout &&
      task.getFromChanges() &&
      constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML !== dataConvert.formatTo &&
      !formatChecker.isOOXFormat(dataConvert.formatTo) &&
      !formatChecker.isBrowserEditorFormat(dataConvert.formatTo) &&
      !cmd.getWopiParams();
    if (canRollback) {
      ctx.logger.warn(
        'rollback to save changes to ooxml. See assemblyFormatAsOrigin param. formatTo=%s',
        formatChecker.getStringFromFormat(dataConvert.formatTo)
      );
      const extOld = path.extname(dataConvert.fileTo);
      const extNew = '.' + formatChecker.getStringFromFormat(constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML);
      dataConvert.formatTo = constants.AVS_OFFICESTUDIO_FILE_OTHER_OOXML;
      dataConvert.fileTo = dataConvert.fileTo.slice(0, -extOld.length) + extNew;
      ({childRes, isTimeout} = yield* spawnProcess(ctx, builderParams, tempDirs, dataConvert, authorProps, getTaskTime, task, isInJwtToken));
    }
    if (clientStatsD) {
      clientStatsD.timing('conv.spawnSync', new Date() - curDate);
      curDate = new Date();
    }
  }
  const resData = yield* postProcess(ctx, cmd, dataConvert, tempDirs, childRes, error, isTimeout);
  ctx.logger.debug('postProcess');
  if (clientStatsD) {
    clientStatsD.timing('conv.postProcess', new Date() - curDate);
    curDate = new Date();
  }
  if (tempDirs) {
    fs.rmSync(tempDirs.temp, {recursive: true, force: true});
    ctx.logger.debug('deleteFolderRecursive');
    if (clientStatsD) {
      clientStatsD.timing('conv.deleteFolderRecursive', new Date() - curDate);
      curDate = new Date();
    }
  }
  if (clientStatsD) {
    clientStatsD.timing('conv.allconvert', new Date() - startDate);
  }
  ctx.logger.info('End Task');
  return resData;
}
function ackTask(ctx, res, task, ack) {
  return co(function* () {
    try {
      if (!res) {
        res = createErrorResponse(ctx, task);
      }
      if (res) {
        yield queue.addResponse(res);
        ctx.logger.info('ackTask addResponse');
      }
    } catch (err) {
      ctx.logger.error('ackTask %s', err.stack);
    } finally {
      ack();
      ctx.logger.info('ackTask ack');
    }
  });
}
function receiveTaskSetTimeout(ctx, task, ack, outParams) {
  //add DownloadTimeout to upload results
  const delay = task.getVisibilityTimeout() * 1000 + ms(cfgDownloadTimeout.wholeCycle);
  return setTimeout(() => {
    return co(function* () {
      outParams.isAck = true;
      ctx.logger.error('receiveTask timeout %d', delay);
      yield ackTask(ctx, null, task, ack);
      yield queue.closeOrWait();
      process.exit(1);
    });
  }, delay);
}
function receiveTask(data, ack) {
  return co(function* () {
    let res = null;
    let task = null;
    const outParams = {isAck: false};
    let timeoutId = undefined;
    const ctx = new operationContext.Context();
    try {
      task = new commonDefines.TaskQueueData(JSON.parse(data));
      if (task) {
        ctx.initFromTaskQueueData(task);
        yield ctx.initTenantCache();
        timeoutId = receiveTaskSetTimeout(ctx, task, ack, outParams);
        res = yield* ExecuteTask(ctx, task);
      }
    } catch (err) {
      ctx.logger.error('receiveTask %s', err.stack);
    } finally {
      clearTimeout(timeoutId);
      if (!outParams.isAck) {
        yield ackTask(ctx, res, task, ack);
      }
    }
  });
}
function createErrorResponse(ctx, task) {
  if (!task) {
    return null;
  }
  ctx.logger.debug('createErrorResponse');
  //simulate error response
  const cmd = task.getCmd();
  cmd.setStatusInfo(constants.CONVERT_TEMPORARY);
  const res = new commonDefines.TaskQueueData();
  res.setCtx(ctx);
  res.setCmd(cmd);
  return res;
}
function simulateErrorResponse(data) {
  const task = new commonDefines.TaskQueueData(JSON.parse(data));
  const ctx = new operationContext.Context();
  ctx.initFromTaskQueueData(task);
  //todo
  //yield ctx.initTenantCache();
  return createErrorResponse(ctx, task);
}
function run() {
  queue = new queueService(simulateErrorResponse);
  queue.on('task', receiveTask);
  queue.init(true, true, true, false, false, false, err => {
    if (null != err) {
      operationContext.global.logger.error('createTaskQueue error: %s', err.stack);
    }
  });
}
exports.run = run;
