gulpfile.js

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

'use strict';

const gulp = require('gulp');

const fs = require('fs');
const nls = require('vscode-nls-dev');
const path = require('path');
const minimist = require('minimist');
const es = require('event-stream');
const sourcemaps = require('gulp-sourcemaps');
const ts = require('gulp-typescript');
const typescript = require('typescript');
const tsProject = ts.createProject('./tsconfig.json', { typescript });
const filter = require('gulp-filter');
const vinyl = require('vinyl');
const parse5 = require('parse5');
const traverse = require('parse5-traverse');
const jsonc = require('comment-json'); // Used to allow comments in nativeStrings.json
const crypto = require('crypto');
const https = require('https');

// Patterns to find HTML files
const htmlFilesPatterns = [
    "ui/**/*.html",
    "Reinstalling the Extension.md"
];

// HTML files for walkthroughs are handled differently, as localization support
// requires specific file name patterns, and must all reside in the same directory.
const walkthroughHtmlFilesPatterns = [
    "walkthrough/**/*.md"
];

const jsonSchemaFilesPatterns = [
    "*.schema.json"
];

const languages = [
    { id: "zh-tw", folderName: "cht", transifexId: "zh-hant" },
    { id: "zh-cn", folderName: "chs", transifexId: "zh-hans" },
    { id: "fr", folderName: "fra" },
    { id: "de", folderName: "deu" },
    { id: "it", folderName: "ita" },
    { id: "es", folderName: "esn" },
    { id: "ja", folderName: "jpn" },
    { id: "ko", folderName: "kor" },
    { id: "ru", folderName: "rus" },
    //{ id: "bg", folderName: "bul" }, // VS Code supports Bulgarian, but VS is not currently localized for it
    //{ id: "hu", folderName: "hun" }, // VS Code supports Hungarian, but VS is not currently localized for it
    { id: "pt-br", folderName: "ptb", transifexId: "pt-BR" },
    { id: "tr", folderName: "trk" },
    { id: "cs", folderName: "csy" },
    { id: "pl", folderName: "plk" }
];


// ****************************
// Command: translations-export
// The following is used to export and XLF file containing english strings for translations.
// The result will be written to: ../vscode-extensions-localization-export/ms-vscode/
// ****************************

const translationProjectName = "vscode-extensions";
const translationExtensionName = "vscode-cpptools";

function removePathPrefix(path, prefix) {
    if (!prefix) {
        return path;
    }
    if (!path.startsWith(prefix)) {
        return path;
    }
    if (path === prefix) {
        return "";
    }
    let ch = prefix.charAt(prefix.length - 1);
    if (ch === '/' || ch === '\\') {
        return path.substring(prefix.length);
    }
    ch = path.charAt(prefix.length);
    if (ch === '/' || ch === '\\') {
        return path.substring(prefix.length + 1);
    }
    return path;
}

const dataLocIdAttribute = "data-loc-id";
const dataLocHintAttribute = "data-loc-hint";

// Helper to traverse HTML tree
// nodeCallback(locId, locHint, node) is invoked for nodes
// attributeCallback(locId, locHint, attribute) is invoked for attributes
const traverseHtml = (contents, nodeCallback, attributeCallback, isFragment) => {
    const htmlTree = isFragment ? parse5.parseFragment(contents) : parse5.parse(contents);
    traverse(htmlTree, {
        pre(node, parent) {
            if (node.attrs) {
                // Check if content text should be localized based on presense of data-loc-id attribute
                let locId = node.attrs.find(attribute => attribute.name.toLowerCase() == dataLocIdAttribute);
                if (locId) {
                    let locHint = node.attrs.find(attribute => attribute.name.toLowerCase() == dataLocHintAttribute);
                    nodeCallback(locId.value, locHint?.value, node);
                }
                // Check if an attribute should be localized based on presense of data-loc-id-<attribute_name> attribute
                node.attrs.forEach(attribute => {
                    if (attribute.name.startsWith(`${dataLocIdAttribute}-`)) {
                        let targetAttributeName = attribute.name.substring(dataLocIdAttribute.length + 1);
                        let targetAttribute = node.attrs.find(a => a.name == targetAttributeName);
                        if (targetAttribute) {
                            let hint = node.attrs.find(a => a.name.toLowerCase() == `${dataLocHintAttribute}-${targetAttributeName}`);
                            attributeCallback(attribute.value, hint?.value, targetAttribute);
                        }
                    }
                });
            }
        }
    });
    return htmlTree;
};

// Traverses the HTML document looking for node and attributes containing data-loc-id, to localize
// Outputs *.nls.json files containing strings to localize.
const processHtmlFiles = () => {
    return es.through(function (file) {
        let localizationJsonContents = {};
        let localizationMetadataContents = {
            messages: [],
            keys: [],
            filePath: removePathPrefix(file.path, file.cwd)
        };
        let nodeCallback = (locId, locHint, node) => {
            let subNodeCount = 0;
            let text = "";
            node.childNodes.forEach((childNode) => {
                if (childNode.nodeName == "#text") {
                    text += childNode.value;
                } else {
                    text += `{${subNodeCount++}}`;
                }
            });
            localizationJsonContents[locId] = text;
            localizationMetadataContents.keys.push(locHint ? { key: locId, comment: [locHint] } : locId);
            localizationMetadataContents.messages.push(text);
        };
        let attributeCallback = (locId, locHint, attribute) => {
            localizationJsonContents[locId] = attribute.value;
            localizationMetadataContents.keys.push(locHint ? { key: locId, comment: [locHint] } : locId);
            localizationMetadataContents.messages.push(attribute.value);
        };
        traverseHtml(file.contents.toString(), nodeCallback, attributeCallback, false);
        this.queue(new vinyl({
            path: path.join(file.path + '.nls.json'),
            contents: Buffer.from(JSON.stringify(localizationJsonContents, null, '\t'), 'utf8')
        }));
        this.queue(new vinyl({
            path: path.join(file.path + '.nls.metadata.json'),
            contents: Buffer.from(JSON.stringify(localizationMetadataContents, null, '\t'), 'utf8')
        }));
    });
};

// descriptionCallback(path, value, parent) is invoked for attributes
const traverseJson = (jsonTree, descriptionCallback, prefixPath) => {
    for (let fieldName in jsonTree) {
        if (jsonTree[fieldName] !== null) {
            if (typeof (jsonTree[fieldName]) == "string" && (fieldName === "description" || fieldName === "markdownDescription")) {
                descriptionCallback(prefixPath, jsonTree[fieldName], jsonTree);
            } else if (typeof (jsonTree[fieldName]) == "object") {
                let path = prefixPath;
                if (path !== "")
                    path = path + ".";
                path = path + fieldName;
                traverseJson(jsonTree[fieldName], descriptionCallback, path);
            }
        }
    }
};

// Traverses schema json files looking for "description" fields to localized.
// The path to the "description" field is used to create a localization key.
const processJsonSchemaFiles = () => {
    return es.through(function (file) {
        let jsonTree = JSON.parse(file.contents.toString());
        let localizationJsonContents = {};
        let filePath = removePathPrefix(file.path, file.cwd);
        let localizationMetadataContents = {
            messages: [],
            keys: [],
            filePath: filePath
        };
        let descriptionCallback = (path, value, parent) => {
            let locId = filePath + "." + path;
            let locHint = parent.descriptionHint;
            localizationJsonContents[locId] = value;
            localizationMetadataContents.keys.push(locHint ? { key: locId, comment: [locHint] } : locId);
            localizationMetadataContents.messages.push(value);
        };
        traverseJson(jsonTree, descriptionCallback, "");
        this.queue(new vinyl({
            path: path.join(file.path + '.nls.json'),
            contents: Buffer.from(JSON.stringify(localizationJsonContents, null, '\t'), 'utf8')
        }));
        this.queue(new vinyl({
            path: path.join(file.path + '.nls.metadata.json'),
            contents: Buffer.from(JSON.stringify(localizationMetadataContents, null, '\t'), 'utf8')
        }));
    });
};

gulp.task("translations-export", (done) => {

    // Transpile the TS to JS, and let vscode-nls-dev scan the files for calls to localize.
    let jsStream = tsProject.src()
        .pipe(sourcemaps.init())
        .pipe(tsProject()).js
        .pipe(nls.createMetaDataFiles());

    // Scan html files for tags with the data-loc-id attribute
    let htmlStream = gulp.src([...htmlFilesPatterns, ...walkthroughHtmlFilesPatterns])
        .pipe(processHtmlFiles());

    let jsonSchemaStream = gulp.src(jsonSchemaFilesPatterns)
        .pipe(processJsonSchemaFiles());

    // Merge files from all source streams
    es.merge(jsStream, htmlStream, jsonSchemaStream)

        // Filter down to only the files we need
        .pipe(filter(['**/*.nls.json', '**/*.nls.metadata.json']))

        // Consoldate them into nls.metadata.json, which the xlf is built from.
        .pipe(nls.bundleMetaDataFiles('ms-vscode.cpptools', '.'))

        // filter down to just the resulting metadata files
        .pipe(filter(['**/nls.metadata.header.json', '**/nls.metadata.json']))

        // Add package.nls.json, used to localized package.json
        .pipe(gulp.src(["package.nls.json"]))

        // package.nls.json and nls.metadata.json are used to generate the xlf file
        // Does not re-queue any files to the stream.  Outputs only the XLF file
        .pipe(nls.createXlfFiles(translationProjectName, translationExtensionName))
        .pipe(gulp.dest(path.join("..", `${translationProjectName}-localization-export`)))
        .pipe(es.wait(() => {
            done();
        }));
});


// ****************************
// Command: translations-import
// The following is used to import an XLF file containing all language strings.
// This results in a i18n directory, which should be checked in.
// ****************************

// Imports translations from raw localized MLCP strings to VS Code .i18n.json files
gulp.task("translations-import", (done) => {
    let options = minimist(process.argv.slice(2), {
        string: "location",
        default: {
            location: "../vscode-translations-import"
        }
    });
    es.merge(languages.map((language) => {
        let id = language.transifexId || language.id;
        return gulp.src(path.join(options.location, id, translationProjectName, `${translationExtensionName}.xlf`))
            .pipe(nls.prepareJsonFiles())
            .pipe(gulp.dest(path.join("./i18n", language.folderName)))
            .pipe(es.wait()); // This is required or it gives `this.pipeTo.end is not a function`.
    }))
        .pipe(es.wait(() => {
            done();
        }));
});

// ****************************
// Command: translations-generate
// The following is used to import an i18n directory structure and generate files used at runtime.
// ****************************

// Generate package.nls.*.json files from: ./i18n/*/package.i18n.json
// Outputs to root path, as these nls files need to be along side package.json
const generateAdditionalLocFiles = () => {
    return gulp.src(['package.nls.json'])
        .pipe(nls.createAdditionalLanguageFiles(languages, 'i18n'))
        .pipe(gulp.dest('.'));
};

// Generates ./dist/nls.bundle.<language_id>.json from files in ./i18n/** *//<src_path>/<filename>.i18n.json
// Localized strings are read from these files at runtime.
const generateSrcLocBundle = () => {
    // Transpile the TS to JS, and let vscode-nls-dev scan the files for calls to localize.
    return tsProject.src()
        .pipe(sourcemaps.init())
        .pipe(tsProject()).js
        .pipe(nls.createMetaDataFiles())
        .pipe(nls.createAdditionalLanguageFiles(languages, "i18n"))
        .pipe(nls.bundleMetaDataFiles('ms-vscode.cpptools', 'dist'))
        .pipe(nls.bundleLanguageFiles())
        .pipe(filter(['**/nls.bundle.*.json', '**/nls.metadata.header.json', '**/nls.metadata.json']))
        .pipe(gulp.dest('dist'));
};

const generateLocalizedHtmlFilesImpl = (file, relativePath, language, isFragment) => {
    let stringTable = {};
    // Try to open i18n file for this file
    let locFile = path.join("./i18n", language.folderName, relativePath + ".i18n.json");
    if (fs.existsSync(locFile)) {
        stringTable = jsonc.parse(fs.readFileSync(locFile).toString());
    }
    // Entire file is scanned and modified, then serialized for that language.
    // Even if no translations are available, we still write new files to dist/html/...

    // Rewrite child nodes to fill in {0}, {1}, etc., in localized string.
    let nodeCallback = (locId, locHint, node) => {
        let locString = stringTable[locId];
        if (locString) {
            let nonTextChildNodes = node.childNodes.filter(childNode => childNode.nodeName != "#text");
            let textParts = locString.split(/\{[0-9]+\}/);
            let matchParts = locString.match(/\{[0-9]+\}/g);
            let newChildNodes = [];
            let i = 0;
            for (; i < textParts.length - 1; i++) {
                if (textParts[i] != "") {
                    newChildNodes.push({ nodeName: "#text", value: textParts[i] });
                }
                let childIndex = matchParts[i].match(/[0-9]+/);
                newChildNodes.push(nonTextChildNodes[childIndex]);
            }
            if (textParts[i] != "") {
                newChildNodes.push({ nodeName: "#text", value: textParts[i] });
            }
            node.childNodes = newChildNodes;
        }
    };
    let attributeCallback = (locId, locHint, attribute) => {
        let value = stringTable[locId];
        if (value) {
            attribute.value = value;
        }
    };
    let htmlTree = traverseHtml(file.contents.toString(), nodeCallback, attributeCallback, isFragment);
    return parse5.serialize(htmlTree);
};

const generateLocalizedHtmlFiles = () => {
    return es.through(function (file) {
        let relativePath = removePathPrefix(file.path, file.cwd);
        languages.map((language) => {
            let newContent = generateLocalizedHtmlFilesImpl(file, relativePath, language, false);
            this.queue(new vinyl({
                path: path.join("html", language.id, relativePath),
                contents: Buffer.from(newContent, 'utf8')
            }));
        });
    });
};

const generateLocalizedWalkthroughHtmlFiles = () => {
    return es.through(function (file) {
        let relativePath = removePathPrefix(file.path, file.cwd);
        languages.map((language) => {
            let newPath = relativePath.substring(0, relativePath.lastIndexOf(".")) + `.nls.${language.id}.md`;
            let newContent = generateLocalizedHtmlFilesImpl(file, relativePath, language, true);
            this.queue(new vinyl({
                path: newPath,
                contents: Buffer.from(newContent, 'utf8')
            }));
        });
        // Put the original in an 'en' file.
        let newPath = relativePath.substring(0, relativePath.lastIndexOf(".")) + ".nls.en.md";
        this.queue(new vinyl({
            path: newPath,
            contents: file.contents
        }));
    });
}

// Generate localized versions of HTML files.
// Check for corresponding localized json file in i18n.
// Generate new version of the HTML file in: dist/html/<language_id>/<path>
const generateHtmlLoc = () => {
    return gulp.src(htmlFilesPatterns)
        .pipe(generateLocalizedHtmlFiles())
        .pipe(gulp.dest('dist'));
};

// Generate localized versions of walkthrough HTML (.md) files.
// Check for corresponding localized json file in i18n.
// Generate new version of the HTML file in: dist/<path>
// The destination filename will have ".md" extension replaced with: .nls.<language>.md
// For example, the Spanish translation of "walkthrough/doc.md" will be written to "dist/walkthrough/doc.nls.es.md".
const generateWalkthroughHtmlLoc = () => {
    return gulp.src(walkthroughHtmlFilesPatterns)
        .pipe(generateLocalizedWalkthroughHtmlFiles())
        .pipe(gulp.dest('dist'));
};

const generateLocalizedJsonSchemaFiles = () => {
    return es.through(function (file) {
        let jsonTree = JSON.parse(file.contents.toString());
        languages.map((language) => {
            let stringTable = {};
            // Try to open i18n file for this file
            let relativePath = removePathPrefix(file.path, file.cwd);
            let locFile = path.join("./i18n", language.folderName, relativePath + ".i18n.json");
            if (fs.existsSync(locFile)) {
                stringTable = jsonc.parse(fs.readFileSync(locFile).toString());
            }
            // Entire file is scanned and modified, then serialized for that language.
            // Even if no translations are available, we still write new files to dist/schema/...
            let keyPrefix = relativePath + ".";
            keyPrefix = keyPrefix.replace(/\\/g, "/");
            let descriptionCallback = (path, value, parent) => {
                if (stringTable[keyPrefix + path]) {
                    if (!parent.markdownDescription)
                        parent.description = stringTable[keyPrefix + path];
                    else
                        parent.markdownDescription = stringTable[keyPrefix + path];
                }
            };
            traverseJson(jsonTree, descriptionCallback, "");
            let newContent = JSON.stringify(jsonTree, null, '\t');
            this.queue(new vinyl({
                path: path.join("schema", language.id, relativePath),
                contents: Buffer.from(newContent, 'utf8')
            }));
        });
    });
};

// Generate localized versions of JSON schema files
// Check for corresponding localized json file in i18n
// Generate new version of the JSON schema file in dist/schema/<language_id>/<path>
const generateJsonSchemaLoc = () => {
    return gulp.src(jsonSchemaFilesPatterns)
        .pipe(generateLocalizedJsonSchemaFiles())
        .pipe(gulp.dest('dist'));
};

gulp.task('translations-generate', gulp.series(generateSrcLocBundle, generateAdditionalLocFiles, generateHtmlLoc, generateWalkthroughHtmlLoc, generateJsonSchemaLoc));