diff --git a/translate/automl/automlTranslationDataset.js b/translate/automl/automlTranslationDataset.js new file mode 100755 index 00000000000..c1412c76cb9 --- /dev/null +++ b/translate/automl/automlTranslationDataset.js @@ -0,0 +1,369 @@ +/** + * Copyright 2018, Google, LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This application demonstrates how to perform basic operations on dataset + * with the Google AutoML Translation API. + * + * For more information, see the documentation at + * https://cloud.google.com/translate/automl/docs + */ + +`use strict`; + +function createDataset(projectId, computeRegion, datasetName, source, target) { + // [START automl_translation_create_dataset] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const datasetName = `name of the dataset to create, e.g. “myDataset”`; + // const source = `source language code, e.g. "en" `; + // const target = `target language code, e.g. "ja" `; + + // A resource that represents Google Cloud Platform location. + const projectLocation = client.locationPath(projectId, computeRegion); + + // Specify the source and target language. + const datasetSpec = { + sourceLanguageCode: source, + targetLanguageCode: target, + }; + + // Set dataset name and dataset specification. + const dataset = { + displayName: datasetName, + translationDatasetMetadata: datasetSpec, + }; + + // Create a dataset with the dataset specification in the region. + client + .createDataset({parent: projectLocation, dataset: dataset}) + .then(responses => { + const dataset = responses[0]; + + // Display the dataset information + console.log(`Dataset name: ${dataset.name}`); + console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`); + console.log(`Dataset display name: ${dataset.displayName}`); + console.log(`Dataset example count: ${dataset.exampleCount}`); + console.log(`Translation dataset specification:`); + console.log( + `\tSource language code: ${ + dataset.translationDatasetMetadata.sourceLanguageCode + }` + ); + console.log( + `\tTarget language code: ${ + dataset.translationDatasetMetadata.targetLanguageCode + }` + ); + console.log(`Dataset create time:`); + console.log(`\tseconds: ${dataset.createTime.seconds}`); + console.log(`\tnanos: ${dataset.createTime.nanos}`); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_create_dataset] +} + +function listDatasets(projectId, computeRegion, filter) { + // [START automl_translation_list_datasets] + const automl = require(`@google-cloud/automl`); + + const client = new automl.v1beta1.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const filter = `filter expressions, must specify field e.g. “imageClassificationModelMetadata:*”`; + + // A resource that represents Google Cloud Platform location. + const projectLocation = client.locationPath(projectId, computeRegion); + + // List all the datasets available in the region by applying filter. + client + .listDatasets({parent: projectLocation, filter: filter}) + .then(responses => { + const datasets = responses[0]; + + // Display the dataset information. + console.log(`List of datasets:`); + datasets.forEach(dataset => { + console.log(`Dataset name: ${dataset.name}`); + console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`); + console.log(`Dataset display name: ${dataset.displayName}`); + console.log(`Dataset example count: ${dataset.exampleCount}`); + console.log(`Translation dataset specification:`); + console.log( + `\tSource language code: ${ + dataset.translationDatasetMetadata.sourceLanguageCode + }` + ); + console.log( + `\tTarget language code: ${ + dataset.translationDatasetMetadata.targetLanguageCode + }` + ); + console.log(`Dataset create time:`); + console.log(`\tseconds: ${dataset.createTime.seconds}`); + console.log(`\tnanos: ${dataset.createTime.nanos}`); + }); + }) + .catch(err => { + console.error(err); + }); + // [START automl_translation_list_datasets] +} + +function getDataset(projectId, computeRegion, datasetId) { + // [START automl_translation_get_dataset] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const datasetId = `Id of the dataset`; + + // Get the full path of the dataset. + const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId); + + // Get complete detail of the dataset. + client + .getDataset({name: datasetFullId}) + .then(responses => { + const dataset = responses[0]; + + // Display the dataset information. + console.log(`Dataset name: ${dataset.name}`); + console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`); + console.log(`Dataset display name: ${dataset.displayName}`); + console.log(`Dataset example count: ${dataset.exampleCount}`); + console.log(`Translation dataset specification:`); + console.log( + `\tSource language code: ${ + dataset.translationDatasetMetadata.sourceLanguageCode + }` + ); + console.log( + `\tTarget language code: ${ + dataset.translationDatasetMetadata.targetLanguageCode + }` + ); + console.log(`Dataset create time:`); + console.log(`\tseconds: ${dataset.createTime.seconds}`); + console.log(`\tnanos: ${dataset.createTime.nanos}`); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_get_dataset] +} + +function importData(projectId, computeRegion, datasetId, path) { + // [START automl_translation_import_data] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const datasetId = `Id of the dataset`; + // const path = `string or array of .csv paths in AutoML Vision CSV format, e.g. “gs://myproject/mytraindata.csv”;` + + // Get the full path of the dataset. + const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId); + + // Get the multiple Google Cloud Storage URIs. + const inputUris = path.split(`,`); + const inputConfig = { + gcsSource: { + inputUris: inputUris, + }, + }; + + // Import data from the input URI. + client + .importData({name: datasetFullId, inputConfig: inputConfig}) + .then(responses => { + const operation = responses[0]; + console.log(`Processing import...`); + return operation.promise(); + }) + .then(responses => { + // The final result of the operation. + if (responses[2].done === true) { + console.log(`Data imported.`); + } + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_import_data] +} + +function deleteDataset(projectId, computeRegion, datasetId) { + // [START automl_translation_delete_dataset] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const datasetId = `Id of the dataset`; + + // Get the full path of the dataset. + const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId); + + // Delete a dataset. + client + .deleteDataset({name: datasetFullId}) + .then(responses => { + const operation = responses[0]; + return operation.promise(); + }) + .then(responses => { + // The final result of the operation. + if (responses[2].done === true) console.log(`Dataset deleted.`); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_delete_dataset] +} + +require(`yargs`) + .demand(1) + .options({ + computeRegion: { + alias: `c`, + type: `string`, + default: process.env.REGION_NAME, + requiresArg: true, + description: `region name e.g. "us-central1"`, + }, + datasetName: { + alias: `n`, + type: `string`, + default: `testDataSet`, + requiresArg: true, + description: `Name of the Dataset`, + }, + datasetId: { + alias: `i`, + type: `string`, + requiresArg: true, + description: `Id of the dataset`, + }, + filter: { + alias: `f`, + default: `translationDatasetMetadata:*`, + type: `string`, + requiresArg: true, + description: `Name of the Dataset to search for`, + }, + multilabel: { + alias: `m`, + type: `string`, + default: false, + requiresArg: true, + description: + `Type of the classification problem, ` + + `False - MULTICLASS, True - MULTILABEL.`, + }, + outputUri: { + alias: `o`, + type: `string`, + requiresArg: true, + description: `URI (or local path) to export dataset`, + }, + path: { + alias: `p`, + type: `string`, + global: true, + default: `gs://nodejs-docs-samples-vcm/en-ja.csv`, + requiresArg: true, + description: `URI or local path to input .csv, or array of .csv paths`, + }, + projectId: { + alias: `z`, + type: `number`, + default: process.env.GCLOUD_PROJECT, + requiresArg: true, + description: `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`, + }, + source: { + alias: `s`, + type: `string`, + requiresArg: true, + description: `The source language to be translated from`, + }, + target: { + alias: `t`, + type: `string`, + requiresArg: true, + description: `The target language to be translated to`, + }, + }) + .command(`createDataset`, `creates a new Dataset`, {}, opts => + createDataset( + opts.projectId, + opts.computeRegion, + opts.datasetName, + opts.source, + opts.target + ) + ) + .command(`list-datasets`, `list all Datasets`, {}, opts => + listDatasets(opts.projectId, opts.computeRegion, opts.filter) + ) + .command(`get-dataset`, `Get a Dataset`, {}, opts => + getDataset(opts.projectId, opts.computeRegion, opts.datasetId) + ) + .command(`delete-dataset`, `Delete a dataset`, {}, opts => + deleteDataset(opts.projectId, opts.computeRegion, opts.datasetId) + ) + .command(`import-data`, `Import labeled items into dataset`, {}, opts => + importData(opts.projectId, opts.computeRegion, opts.datasetId, opts.path) + ) + .example(`node $0 create-dataset -n "newDataSet" -s "en" -t "ja"`) + .example(`node $0 list-datasets -f "translationDatasetMetadata:*"`) + .example(`node $0 get-dataset -i "DATASETID"`) + .example(`node $0 delete-dataset -i "DATASETID"`) + .example( + `node $0 import-data -i "dataSetId" -p "gs://myproject/mytraindata.csv"` + ) + .wrap(120) + .recommendCommands() + .help() + .strict().argv; diff --git a/translate/automl/automlTranslationModel.js b/translate/automl/automlTranslationModel.js new file mode 100755 index 00000000000..09caec34986 --- /dev/null +++ b/translate/automl/automlTranslationModel.js @@ -0,0 +1,447 @@ +/** + * Copyright 2018, Google, LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This application demonstrates how to perform basic operations on model + * with the Google AutoML Translation API. + * + * For more information, see the documentation at + * https://cloud.google.com/translate/automl/docs + */ + +`use strict`; + +function createModel(projectId, computeRegion, datasetId, modelName) { + // [START automl_translation_create_model] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const datasetId = `Id of the dataset`; + // const modelName = `Name of the model, e.g. "myModel"`; + + // A resource that represents Google Cloud Platform location. + const projectLocation = client.locationPath(projectId, computeRegion); + + // Set model name and dataset. + const myModel = { + displayName: modelName, + datasetId: datasetId, + translationModelMetadata: {}, + }; + + // Create a model with the model metadata in the region. + client + .createModel({parent: projectLocation, model: myModel}) + .then(responses => { + const operation = responses[0]; + const initialApiResponse = responses[1]; + + console.log(`Training operation name: `, initialApiResponse.name); + console.log(`Training started...`); + return operation.promise(); + }) + .then(responses => { + // The final result of the operation. + const model = responses[0]; + console.log(model); + // Retrieve deployment state. + let deploymentState = ``; + if (model.deploymentState === 1) { + deploymentState = `deployed`; + } else if (model.deploymentState === 2) { + deploymentState = `undeployed`; + } + + // Display the model information. + console.log(`Model name: ${model.name}`); + console.log(`Model id: ${model.name.split(`/`).pop(-1)}`); + console.log(`Model display name: ${model.displayName}`); + console.log(`Model create time:`); + console.log(`\tseconds: ${model.createTime.seconds}`); + console.log(`\tnanos: ${model.createTime.nanos}`); + console.log(`Model deployment state: ${deploymentState}`); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_create_model] +} + +function listModels(projectId, computeRegion, filter) { + // [START automl_translation_list_models] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const filter = `filter expressions, must specify field, e.g. "translationDatasetMetadata:*”`; + + // A resource that represents Google Cloud Platform location. + const projectLocation = client.locationPath(projectId, computeRegion); + + // List all the models available in the region by applying filter. + client + .listModels({parent: projectLocation, filter: filter}) + .then(responses => { + const models = responses[0]; + + // Display the model information. + console.log(`List of models:`); + models.forEach(model => { + console.log(`Model name: ${model.name}`); + console.log(`Model id: ${model.name.split(`/`).pop(-1)}`); + console.log(`Model display name: ${model.displayName}`); + console.log(`Model dataset id: ${model.datasetId}`); + console.log(`Model create time:`); + console.log(`\tseconds: ${model.createTime.seconds}`); + console.log(`\tnanos: ${model.createTime.nanos}`); + console.log(`Model update time:`); + console.log(`\tseconds: ${model.updateTime.seconds}`); + console.log(`\tnanos: ${model.updateTime.nanos}`); + console.log(`Model deployment state: ${model.deploymentState}`); + console.log(`\n`); + }); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_list_models] +} + +function getModel(projectId, computeRegion, modelId) { + // [START automl_translation_get_model] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const modelId = `id of the model, e.g. “ICN12345”`; + + // Get the full path of the model. + const modelFullId = client.modelPath(projectId, computeRegion, modelId); + + // Get complete detail of the model. + client + .getModel({name: modelFullId}) + .then(responses => { + const model = responses[0]; + + // Display the model information. + console.log(`Model name: ${model.name}`); + console.log(`Model id: ${model.name.split(`/`).pop(-1)}`); + console.log(`Model display name: ${model.displayName}`); + console.log(`Model dataset id: ${model.datasetId}`); + if (model.modelMetadata === `translationModelMetadata`) { + console.log(`Translation model metadata:`); + console.log( + `\tBase model: ${model.translationModelMetadata.baseModel}` + ); + console.log( + `\tSource language code: ${ + model.translationModelMetadata.sourceLanguageCode + }` + ); + console.log( + `\tTarget language code: ${ + model.translationModelMetadata.targetLanguageCode + }` + ); + } else if (model.modelMetadata === `textClassificationModelMetadata`) { + console.log( + `Text classification model metadata: ${ + model.textClassificationModelMetadata + }` + ); + } else if (model.modelMetadata === `imageClassificationModelMetadata`) { + console.log(`Image classification model metadata:`); + console.log( + `\tBase model id: ${ + model.imageClassificationModelMetadata.baseModelId + }` + ); + console.log( + `\tTrain budget: ${ + model.imageClassificationModelMetadata.trainBudget + }` + ); + console.log( + `\tTrain cost: ${model.imageClassificationModelMetadata.trainCost}` + ); + console.log( + `\tStop reason: ${model.imageClassificationModelMetadata.stopReason}` + ); + } + console.log(`Model create time:`); + console.log(`\tseconds: ${model.createTime.seconds}`); + console.log(`\tnanos: ${model.createTime.nanos}`); + console.log(`Model update time:`); + console.log(`\tseconds: ${model.updateTime.seconds}`); + console.log(`\tnanos: ${model.updateTime.nanos}`); + console.log(`Model deployment state: ${model.deploymentState}`); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_get_model] +} + +function listModelEvaluations(projectId, computeRegion, modelId, filter) { + // [START automl_translation_list_model_evaluations] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const modelId = `id of the model, e.g. “ICN12345”`; + // const filter = `filter expressions, must specify field, e.g. “imageClassificationModelMetadata:*”`; + + // Get the full path of the model. + const modelFullId = client.modelPath(projectId, computeRegion, modelId); + + // List all the model evaluations in the model by applying filter. + client + .listModelEvaluations({parent: modelFullId, filter: filter}) + .then(responses => { + const elements = responses[0]; + console.log(`List of model evaluations:`); + elements.forEach(element => { + console.log(element); + }); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_list_model_evaluations] +} + +function getModelEvaluation( + projectId, + computeRegion, + modelId, + modelEvaluationId +) { + // [START automl_translation_get_model_evaluation] + const automl = require(`@google-cloud/automl`); + + const client = new automl.v1beta1.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const modelId = `id of the model, e.g. “ICN12345”`; + // const modelEvaluationId = `Id of your model evaluation, e.g “ICN12345” + + // Get the full path of the model evaluation. + const modelEvaluationFullId = client.modelEvaluationPath( + projectId, + computeRegion, + modelId, + modelEvaluationId + ); + + // Get complete detail of the model evaluation. + client + .getModelEvaluation({name: modelEvaluationFullId}) + .then(responses => { + const response = responses[0]; + console.log(response); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_get_model_evaluation] +} + +function deleteModel(projectId, computeRegion, modelId) { + // [START automl_translation_delete_model] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const modelId = `id of the model, e.g. “ICN12345”`; + + // Get the full path of the model. + const modelFullId = client.modelPath(projectId, computeRegion, modelId); + + // Delete a model. + client + .deleteModel({name: modelFullId}) + .then(responses => { + const operation = responses[0]; + return operation.promise(); + }) + .then(responses => { + // The final result of the operation. + if (responses[2].done === true) console.log(`Model deleted.`); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_delete_model] +} + +function getOperationStatus(operationFullId) { + // [START automl_translation_get_operation_status] + const automl = require(`@google-cloud/automl`).v1beta1; + + const client = new automl.AutoMlClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const operationFullId = `Full name of an operation, eg. “Projects//locations/us-central1/operations/ + + // Get the latest state of a long-running operation. + client.operationsClient.getOperation(operationFullId).then(responses => { + const response = responses[0]; + console.log(`Operation status: ${response}`); + }); + // [END automl_translation_get_operation_status] +} + +require(`yargs`) + .demand(1) + .options({ + computeRegion: { + alias: `c`, + type: `string`, + default: process.env.REGION_NAME, + requiresArg: true, + description: `region name e.g. "us-central1"`, + }, + datasetId: { + alias: `i`, + type: `string`, + requiresArg: true, + description: `Id of the dataset`, + }, + filter: { + alias: `f`, + default: ``, + type: `string`, + requiresArg: true, + description: `Name of the Dataset to search for`, + }, + modelName: { + alias: `m`, + type: `string`, + default: false, + requiresArg: true, + description: `Name of the model`, + }, + modelId: { + alias: `a`, + type: `string`, + default: ``, + requiresArg: true, + description: `Id of the model`, + }, + modelEvaluationId: { + alias: `e`, + type: `string`, + default: ``, + requiresArg: true, + description: `Id of the model evaluation`, + }, + operationFullId: { + alias: `o`, + type: `string`, + default: ``, + requiresArg: true, + description: `Full name of an operation`, + }, + projectId: { + alias: `z`, + type: `number`, + default: process.env.GCLOUD_PROJECT, + requiresArg: true, + description: `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`, + }, + }) + .command(`create-model`, `creates a new Model`, {}, opts => + createModel( + opts.projectId, + opts.computeRegion, + opts.datasetId, + opts.modelName + ) + ) + .command( + `get-operation-status`, + `Gets status of current operation`, + {}, + opts => getOperationStatus(opts.operationFullId) + ) + .command(`list-models`, `list all Models`, {}, opts => + listModels(opts.projectId, opts.computeRegion, opts.filter) + ) + .command(`get-model`, `Get a Model`, {}, opts => + getModel(opts.projectId, opts.computeRegion, opts.modelId) + ) + .command(`list-model-evaluations`, `List model evaluations`, {}, opts => + listModelEvaluations( + opts.projectId, + opts.computeRegion, + opts.modelId, + opts.filter + ) + ) + .command(`get-model-evaluation`, `Get model evaluation`, {}, opts => + getModelEvaluation( + opts.projectId, + opts.computeRegion, + opts.modelId, + opts.modelEvaluationId + ) + ) + .command(`delete-model`, `Delete a Model`, {}, opts => + deleteModel(opts.projectId, opts.computeRegion, opts.modelId) + ) + .example(`node $0 create-model -i "DatasetID" -m "myModelName"`) + .example(`node $0 get-operation-status -i "datasetId" -o "OperationFullID"`) + .example(`node $0 list-models -f "translationModelMetadata:*"`) + .example(`node $0 get-model -a "ModelID"`) + .example(`node $0 list-model-evaluations -a "ModelID"`) + .example(`node $0 get-model-evaluation -a "ModelId" -e "ModelEvaluationID"`) + .example(`node $0 delete-model -a "ModelID"`) + .wrap(120) + .recommendCommands() + .help() + .strict().argv; diff --git a/translate/automl/automlTranslationPredict.js b/translate/automl/automlTranslationPredict.js new file mode 100755 index 00000000000..366660be578 --- /dev/null +++ b/translate/automl/automlTranslationPredict.js @@ -0,0 +1,146 @@ +/** + * Copyright 2018, Google, LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This application demonstrates how to perform basic operations on prediction + * with the Google AutoML Translation API. + * + * For more information, see the documentation at + * https://cloud.google.com/translate/automl/docs + */ + +`use strict`; + +function predict( + projectId, + computeRegion, + modelId, + filePath, + translationAllowFallback +) { + // [START automl_translation_predict] + const automl = require(`@google-cloud/automl`).v1beta1; + const fs = require(`fs`); + + // Create client for prediction service. + const client = new automl.PredictionServiceClient(); + + /** + * TODO(developer): Uncomment the following line before running the sample. + */ + // const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`; + // const computeRegion = `region-name, e.g. "us-central1"`; + // const modelId = `id of the model, e.g. “ICN12345”`; + // const filePath = `local text file path of content to be classified, e.g. "./resources/test.txt"`; + // const translationAllowFallback = `use Google translation model as fallback, e.g. "False" or "True"`; + + // Get the full path of the model. + const modelFullId = client.modelPath(projectId, computeRegion, modelId); + + // Read the file content for translation. + const content = fs.readFileSync(filePath, `utf8`); + + // Set the payload by giving the content of the file. + const payload = { + textSnippet: { + content: content, + }, + }; + + // Params is additional domain-specific parameters. + // TranslationAllowFallback allows to use Google translation model. + let params = {}; + if (translationAllowFallback) { + params = { + translationAllowFallback: true, + }; + } + + client + .predict({ + name: modelFullId, + payload: payload, + params: params, + }) + .then(responses => { + const response = responses[0]; + console.log( + `Translated Content: `, + response.payload[0].translation.translatedContent.content + ); + }) + .catch(err => { + console.error(err); + }); + // [END automl_translation_predict] +} + +require(`yargs`) + .demand(1) + .options({ + computeRegion: { + alias: `c`, + type: `string`, + default: process.env.REGION_NAME, + requiresArg: true, + description: `region name e.g. "us-central1"`, + }, + filePath: { + alias: `f`, + default: `./resources/testInput.txt`, + type: `string`, + requiresArg: true, + description: `local text file path of the content to be classified`, + }, + modelId: { + alias: `i`, + type: `string`, + requiresArg: true, + description: `Id of the model which will be used for text classification`, + }, + projectId: { + alias: `z`, + type: `number`, + default: process.env.GCLOUD_PROJECT, + requiresArg: true, + description: `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`, + }, + translationAllowFallback: { + alias: `t`, + type: `string`, + default: `False`, + requiresArg: true, + description: + `Use true if AutoML will fallback to use a Google translation model for` + + `translation requests if the specified AutoML translation model cannot` + + `serve the request. Use false to not use Google translation model.`, + }, + }) + .command(`predict`, `classify the content`, {}, opts => + predict( + opts.projectId, + opts.computeRegion, + opts.modelId, + opts.filePath, + opts.translationAllowFallback + ) + ) + .example( + `node $0 predict -i "modelId" -f "./resources/testInput.txt" -t "False"` + ) + .wrap(120) + .recommendCommands() + .help() + .strict().argv; diff --git a/translate/automl/package.json b/translate/automl/package.json new file mode 100644 index 00000000000..b24d421a328 --- /dev/null +++ b/translate/automl/package.json @@ -0,0 +1,26 @@ +{ + "name": "automl", + "version": "1.0.0", + "description": "", + "main": "automlTranslationDataset.js", + "scripts": { + "test": "ava -T 1m --verbose system-test/*.test.js" + }, + "engines": { + "node": ">=8" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@google-cloud/automl": "^0.1.1", + "@google-cloud/translate": "^1.1.0", + "yargs": "^12.0.1", + "mathjs": "^5.0.4" + }, + "devDependencies": { + "@google-cloud/nodejs-repo-tools": "^2.3.0", + "ava": "^0.25.0", + "proxyquire": "^2.0.1", + "sinon": "^6.0.1" + } +} diff --git a/translate/automl/resources/testInput.txt b/translate/automl/resources/testInput.txt new file mode 100644 index 00000000000..acea938083b --- /dev/null +++ b/translate/automl/resources/testInput.txt @@ -0,0 +1 @@ +Tell me how this ends diff --git a/translate/automl/system-test/.eslintrc.yml b/translate/automl/system-test/.eslintrc.yml new file mode 100644 index 00000000000..8b2fa88f8e1 --- /dev/null +++ b/translate/automl/system-test/.eslintrc.yml @@ -0,0 +1,5 @@ +--- +rules: + node/no-unpublished-require: off + node/no-unsupported-features: off + no-empty: off \ No newline at end of file diff --git a/translate/automl/system-test/automlTranslation.test.js b/translate/automl/system-test/automlTranslation.test.js new file mode 100644 index 00000000000..931f6a5a7ad --- /dev/null +++ b/translate/automl/system-test/automlTranslation.test.js @@ -0,0 +1,140 @@ +/** + * Copyright 2018, Google, LLC. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const test = require(`ava`); +const tools = require(`@google-cloud/nodejs-repo-tools`); + +const cmdDataset = `node automlTranslationDataset.js`; +const cmdModel = `node automlTranslationModel.js`; +const cmdPredict = `node automlTranslationPredict.js`; + +const testDataSetName = `testDataSet`; +const dummyDataSet = `dummyDataSet`; +const testModelName = 'dummyModel'; +const sampleText = `./resources/testInput.txt`; + +const donotdeleteModelId = `TRL188026453969732486`; + +// Skipped because it's been taking too long to delete datasets +test.skip(`should create a create, list, and delete a dataset`, async t => { + // Check to see that this dataset does not yet exist + let output = await tools.runAsync(`${cmdDataset} list-datasets`); + t.false(output.includes(testDataSetName)); + + // Create dataset + output = await tools.runAsync( + `${cmdDataset} create-dataset -n "${testDataSetName}"` + ); + const dataSetId = output + .split(`\n`)[1] + .split(`:`)[1] + .trim(); + t.true(output.includes(`Dataset display name: ${testDataSetName}`)); + + // Delete dataset + output = await tools.runAsync( + `${cmdDataset} delete-dataset -i "${dataSetId}"` + ); + t.true(output.includes(`Dataset deleted.`)); +}); + +// We make two models running this test, see hard-coded workaround below +test.skip(`should create a dataset, import data, and start making a model`, async t => { + // Check to see that this dataset does not yet exist + let output = await tools.runAsync(`${cmdDataset} list-datasets`); + t.false(output.includes(dummyDataSet)); + + // Create dataset + output = await tools.runAsync( + `${cmdDataset} create-dataset -n "${dummyDataSet}"` + ); + const dataSetId = output + .split(`\n`)[1] + .split(`:`)[1] + .trim(); + t.true(output.includes(`Dataset display name: ${dummyDataSet}`)); + + // Import Data + output = await tools.runAsync( + `${cmdDataset} import-data -i "${dataSetId}" -p "gs://nodejs-docs-samples-vcm/flowerTraindata20lines.csv"` + ); + t.true(output.includes(`Data imported.`)); + + // Check to make sure model doesn't already exist + output = await tools.runAsync(`${cmdModel} list-models`); + t.false(output.includes(`${testModelName}`)); + + // Begin training dataset, getting operation ID for next operation + output = await tools.runAsync( + `${cmdModel} create-model -i "${dataSetId}" -m "${testModelName}" -t "2"` + ); + const operationName = output + .split(`\n`)[0] + .split(`:`)[1] + .trim(); + t.true(output.includes(`Training started...`)); + + // Poll operation status, here confirming that operation is not complete yet + output = await tools.runAsync( + `${cmdModel} get-operation-status -i "${dataSetId}" -o "${operationName}"` + ); + t.true(output.includes(`done: false`)); +}); + +test(`should run get model (from a prexisting model)`, async t => { + // Confirm dataset exists + let output = await tools.runAsync(`${cmdDataset} list-datasets`); + t.true(output.includes(`me_do_not_delete`)); + + // List model evaluations, confirm model exists + output = await tools.runAsync( + `${cmdModel} list-model-evaluations -a "${donotdeleteModelId}"` + ); + t.true(output.includes(`translationEvaluationMetrics:`)); + + // Get model evaluation + output = await tools.runAsync( + `${cmdModel} get-model -a "${donotdeleteModelId}"` + ); + t.true(output.includes(`Model deployment state: DEPLOYED`)); +}); + +test(`should run Prediction from prexisting model`, async t => { + // Confirm dataset exists + let output = await tools.runAsync(`${cmdDataset} list-datasets`); + t.true(output.includes(`me_do_not_delete`)); + + // List model evaluations, confirm model exists + output = await tools.runAsync( + `${cmdModel} list-model-evaluations -a "${donotdeleteModelId}"` + ); + t.true(output.includes(`translationEvaluationMetrics:`)); + + // Run prediction on 'testImage.jpg' in resources folder + output = await tools.runAsync( + `${cmdPredict} predict -i "${donotdeleteModelId}" -f "${sampleText}" -t "False"` + ); + t.true( + output.includes(`Translated Content: これがどのように終わるか教えて`) + ); +}); + +// List datasets +test(`should list datasets`, async t => { + const output = await tools.runAsync(`${cmdDataset} list-datasets`); + t.true(output.includes(`List of datasets:`)); +}); diff --git a/translate/package.json b/translate/package.json index 1c2fff4906d..beff7870be3 100644 --- a/translate/package.json +++ b/translate/package.json @@ -15,6 +15,8 @@ }, "dependencies": { "@google-cloud/translate": "^2.1.2", + "@google-cloud/automl": "^0.1.1", + "mathjs": "^5.1.0", "yargs": "^12.0.1" }, "devDependencies": {