diff --git a/cloud-sql/mysql/sqlalchemy/README.md b/cloud-sql/mysql/sqlalchemy/README.md index 1de0bda3ca37..505f87fe61f2 100644 --- a/cloud-sql/mysql/sqlalchemy/README.md +++ b/cloud-sql/mysql/sqlalchemy/README.md @@ -17,17 +17,6 @@ name. [instructions](https://cloud.google.com/sql/docs/mysql/connect-external-app#4_if_required_by_your_authentication_method_create_a_service_account). Download a JSON key to use to authenticate your connection. -1. Use the information noted in the previous steps: -```bash -export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/key.json -export CLOUD_SQL_CONNECTION_NAME='::' -export DB_USER='my-db-user' -export DB_PASS='my-db-pass' -export DB_NAME='my_db' -``` -Note: Saving credentials in environment variables is convenient, but not secure - consider a more -secure solution such as [Cloud KMS](https://cloud.google.com/kms/) to help keep secrets safe. - ## Running locally To run this application locally, download and install the `cloud_sql_proxy` by @@ -52,6 +41,9 @@ export DB_USER='' export DB_PASS='' export DB_NAME='' ``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. Then use this command to launch the proxy in the background: ```bash @@ -67,6 +59,9 @@ $env:DB_USER="" $env:DB_PASS="" $env:DB_NAME="" ``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. Then use this command to launch the proxy in a separate PowerShell session: ```powershell @@ -87,7 +82,7 @@ sudo chown -R $USER /cloudsql You'll also need to initialize an environment variable containing the directory you just created: ```bash -export DB_SOCKET_PATH=/path/to/the/new/directory +export DB_SOCKET_DIR=/path/to/the/new/directory ``` Use these terminal commands to initialize other environment variables as well: @@ -98,10 +93,13 @@ export DB_USER='' export DB_PASS='' export DB_NAME='' ``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. Then use this command to launch the proxy in the background: ```bash -./cloud_sql_proxy -dir=$DB_SOCKET_PATH --instances=$INSTANCE_CONNECTION_NAME --credential_file=$GOOGLE_APPLICATION_CREDENTIALS & +./cloud_sql_proxy -dir=$DB_SOCKET_DIR --instances=$INSTANCE_CONNECTION_NAME --credential_file=$GOOGLE_APPLICATION_CREDENTIALS & ``` ### Testing the application diff --git a/cloud-sql/mysql/sqlalchemy/app.flexible.yaml b/cloud-sql/mysql/sqlalchemy/app.flexible.yaml index 58c9c0c15428..0423cd607f1d 100644 --- a/cloud-sql/mysql/sqlalchemy/app.flexible.yaml +++ b/cloud-sql/mysql/sqlalchemy/app.flexible.yaml @@ -29,7 +29,8 @@ beta_settings: cloud_sql_instances: ::=tcp: # Remember - storing secrets in plaintext is potentially unsafe. Consider using -# something like https://cloud.google.com/kms/ to help keep secrets secret. +# something like https://cloud.google.com/secret-manager/docs/overview to help keep +# secrets secret. env_variables: CLOUD_SQL_CONNECTION_NAME: :: DB_USER: my-db-user diff --git a/cloud-sql/mysql/sqlalchemy/app.standard.yaml b/cloud-sql/mysql/sqlalchemy/app.standard.yaml index 600d8f9d4759..7fe39ab23049 100644 --- a/cloud-sql/mysql/sqlalchemy/app.standard.yaml +++ b/cloud-sql/mysql/sqlalchemy/app.standard.yaml @@ -15,7 +15,8 @@ runtime: python37 # Remember - storing secrets in plaintext is potentially unsafe. Consider using -# something like https://cloud.google.com/kms/ to help keep secrets secret. +# something like https://cloud.google.com/secret-manager/docs/overview to help keep +# secrets secret. env_variables: CLOUD_SQL_CONNECTION_NAME: :: DB_USER: my-db-user diff --git a/cloud-sql/mysql/sqlalchemy/main.py b/cloud-sql/mysql/sqlalchemy/main.py index cf4ac92b03de..6fca42154e4d 100644 --- a/cloud-sql/mysql/sqlalchemy/main.py +++ b/cloud-sql/mysql/sqlalchemy/main.py @@ -20,13 +20,6 @@ import sqlalchemy -# Remember - storing secrets in plaintext is potentially unsafe. Consider using -# something like https://cloud.google.com/kms/ to help keep secrets secret. -db_user = os.environ.get("DB_USER") -db_pass = os.environ.get("DB_PASS") -db_name = os.environ.get("DB_NAME") -cloud_sql_connection_name = os.environ.get("CLOUD_SQL_CONNECTION_NAME") - app = Flask(__name__) logger = logging.getLogger() @@ -68,59 +61,74 @@ def init_connection_engine(): def init_tcp_connection_engine(db_config): # [START cloud_sql_mysql_sqlalchemy_create_tcp] - db_socket_addr = os.environ.get("DB_HOST").split(":") - - # Extract host and port from socket address - db_host = db_socket_addr[0] - db_port = int(db_socket_addr[1]) - - return sqlalchemy.create_engine( + # Remember - storing secrets in plaintext is potentially unsafe. Consider using + # something like https://cloud.google.com/secret-manager/docs/overview to help keep + # secrets secret. + db_user = os.environ["DB_USER"] + db_pass = os.environ["DB_PASS"] + db_name = os.environ["DB_NAME"] + db_host = os.environ["DB_HOST"] + + # Extract host and port from db_host + host_args = db_host.split(":") + db_hostname, db_port = host_args[0], int(host_args[1]) + + pool = sqlalchemy.create_engine( # Equivalent URL: # mysql+pymysql://:@:/ sqlalchemy.engine.url.URL( drivername="mysql+pymysql", - username=db_user, - password=db_pass, - host=db_host, - port=db_port, - database=db_name, + username=db_user, # e.g. "my-database-user" + password=db_pass, # e.g. "my-database-password" + host=db_hostname, # e.g. "127.0.0.1" + port=db_port, # e.g. 3306 + database=db_name, # e.g. "my-database-name" ), # ... Specify additional properties here. - # [START_EXCLUDE] + # [END cloud_sql_mysql_sqlalchemy_create_tcp] **db_config - # [END_EXCLUDE] + # [START cloud_sql_mysql_sqlalchemy_create_tcp] ) # [END cloud_sql_mysql_sqlalchemy_create_tcp] + return pool + def init_unix_connection_engine(db_config): # [START cloud_sql_mysql_sqlalchemy_create_socket] - if os.environ.get("DB_SOCKET_PATH"): - socket_path = os.environ.get("DB_SOCKET_PATH") - else: - socket_path = "/cloudsql" - - return sqlalchemy.create_engine( + # Remember - storing secrets in plaintext is potentially unsafe. Consider using + # something like https://cloud.google.com/secret-manager/docs/overview to help keep + # secrets secret. + db_user = os.environ["DB_USER"] + db_pass = os.environ["DB_PASS"] + db_name = os.environ["DB_NAME"] + db_socket_dir = os.environ.get("DB_SOCKET_DIR", "/cloudsql") + cloud_sql_connection_name = os.environ["CLOUD_SQL_CONNECTION_NAME"] + + pool = sqlalchemy.create_engine( # Equivalent URL: # mysql+pymysql://:@/?unix_socket=/ sqlalchemy.engine.url.URL( drivername="mysql+pymysql", - username=db_user, - password=db_pass, - database=db_name, + username=db_user, # e.g. "my-database-user" + password=db_pass, # e.g. "my-database-password" + database=db_name, # e.g. "my-database-name" query={ "unix_socket": "{}/{}".format( - socket_path, - cloud_sql_connection_name) + db_socket_dir, # e.g. "/cloudsql" + cloud_sql_connection_name) # i.e "::" } ), # ... Specify additional properties here. - # [START_EXCLUDE] + + # [END cloud_sql_mysql_sqlalchemy_create_socket] **db_config - # [END_EXCLUDE] + # [START cloud_sql_mysql_sqlalchemy_create_socket] ) # [END cloud_sql_mysql_sqlalchemy_create_socket] + return pool + # The SQLAlchemy engine will help manage interactions, including automatically # managing a pool of connections to your database diff --git a/cloud-sql/postgres/sqlalchemy/README.md b/cloud-sql/postgres/sqlalchemy/README.md index 383d1159ab0a..1c385be29804 100644 --- a/cloud-sql/postgres/sqlalchemy/README.md +++ b/cloud-sql/postgres/sqlalchemy/README.md @@ -17,17 +17,6 @@ name. [instructions](https://cloud.google.com/sql/docs/postgres/connect-external-app#4_if_required_by_your_authentication_method_create_a_service_account). Download a JSON key to use to authenticate your connection. -1. Use the information noted in the previous steps: -```bash -export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/key.json -export CLOUD_SQL_CONNECTION_NAME='::' -export DB_USER='my-db-user' -export DB_PASS='my-db-pass' -export DB_NAME='my_db' -``` -Note: Saving credentials in environment variables is convenient, but not secure - consider a more -secure solution such as [Cloud KMS](https://cloud.google.com/kms/) to help keep secrets safe. - ## Running locally To run this application locally, download and install the `cloud_sql_proxy` by @@ -51,6 +40,9 @@ export DB_USER='' export DB_PASS='' export DB_NAME='' ``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. Then use this command to launch the proxy in the background: ```bash @@ -66,6 +58,9 @@ $env:DB_USER="" $env:DB_PASS="" $env:DB_NAME="" ``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. Then use this command to launch the proxy in a separate PowerShell session: ```powershell @@ -85,7 +80,7 @@ sudo chown -R $USER /path/to/the/new/directory You'll also need to initialize an environment variable containing the directory you just created: ```bash -export DB_SOCKET_PATH=/path/to/the/new/directory +export DB_SOCKET_DIR=/path/to/the/new/directory ``` Use these terminal commands to initialize other environment variables as well: @@ -96,10 +91,13 @@ export DB_USER='' export DB_PASS='' export DB_NAME='' ``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. Then use this command to launch the proxy in the background: ```bash -./cloud_sql_proxy -dir=$DB_SOCKET_PATH --instances=$INSTANCE_CONNECTION_NAME --credential_file=$GOOGLE_APPLICATION_CREDENTIALS & +./cloud_sql_proxy -dir=$DB_SOCKET_DIR --instances=$INSTANCE_CONNECTION_NAME --credential_file=$GOOGLE_APPLICATION_CREDENTIALS & ``` ### Testing the application diff --git a/cloud-sql/postgres/sqlalchemy/app.flexible.yaml b/cloud-sql/postgres/sqlalchemy/app.flexible.yaml index 58c9c0c15428..0423cd607f1d 100644 --- a/cloud-sql/postgres/sqlalchemy/app.flexible.yaml +++ b/cloud-sql/postgres/sqlalchemy/app.flexible.yaml @@ -29,7 +29,8 @@ beta_settings: cloud_sql_instances: ::=tcp: # Remember - storing secrets in plaintext is potentially unsafe. Consider using -# something like https://cloud.google.com/kms/ to help keep secrets secret. +# something like https://cloud.google.com/secret-manager/docs/overview to help keep +# secrets secret. env_variables: CLOUD_SQL_CONNECTION_NAME: :: DB_USER: my-db-user diff --git a/cloud-sql/postgres/sqlalchemy/app.standard.yaml b/cloud-sql/postgres/sqlalchemy/app.standard.yaml index 600d8f9d4759..7fe39ab23049 100644 --- a/cloud-sql/postgres/sqlalchemy/app.standard.yaml +++ b/cloud-sql/postgres/sqlalchemy/app.standard.yaml @@ -15,7 +15,8 @@ runtime: python37 # Remember - storing secrets in plaintext is potentially unsafe. Consider using -# something like https://cloud.google.com/kms/ to help keep secrets secret. +# something like https://cloud.google.com/secret-manager/docs/overview to help keep +# secrets secret. env_variables: CLOUD_SQL_CONNECTION_NAME: :: DB_USER: my-db-user diff --git a/cloud-sql/postgres/sqlalchemy/main.py b/cloud-sql/postgres/sqlalchemy/main.py index 4e948f1d68b8..9892bf9a5387 100644 --- a/cloud-sql/postgres/sqlalchemy/main.py +++ b/cloud-sql/postgres/sqlalchemy/main.py @@ -20,13 +20,6 @@ import sqlalchemy -# Remember - storing secrets in plaintext is potentially unsafe. Consider using -# something like https://cloud.google.com/kms/ to help keep secrets secret. -db_user = os.environ.get("DB_USER") -db_pass = os.environ.get("DB_PASS") -db_name = os.environ.get("DB_NAME") -cloud_sql_connection_name = os.environ.get("CLOUD_SQL_CONNECTION_NAME") - app = Flask(__name__) logger = logging.getLogger() @@ -71,60 +64,74 @@ def init_connection_engine(): def init_tcp_connection_engine(db_config): # [START cloud_sql_postgres_sqlalchemy_create_tcp] - db_socket_addr = os.environ.get("DB_HOST").split(":") - - # Extract host and port from socket address - db_host = db_socket_addr[0] - db_port = int(db_socket_addr[1]) - - return sqlalchemy.create_engine( + # Remember - storing secrets in plaintext is potentially unsafe. Consider using + # something like https://cloud.google.com/secret-manager/docs/overview to help keep + # secrets secret. + db_user = os.environ["DB_USER"] + db_pass = os.environ["DB_PASS"] + db_name = os.environ["DB_NAME"] + db_host = os.environ["DB_HOST"] + + # Extract host and port from db_host + host_args = db_host.split(":") + db_hostname, db_port = host_args[0], int(host_args[1]) + + pool = sqlalchemy.create_engine( # Equivalent URL: # postgres+pg8000://:@:/ sqlalchemy.engine.url.URL( drivername="postgres+pg8000", - username=db_user, - password=db_pass, - host=db_host, - port=db_port, - database=db_name + username=db_user, # e.g. "my-database-user" + password=db_pass, # e.g. "my-database-password" + host=db_hostname, # e.g. "127.0.0.1" + port=db_port, # e.g. 5432 + database=db_name # e.g. "my-database-name" ), # ... Specify additional properties here. - # [START_EXCLUDE] + # [END cloud_sql_postgres_sqlalchemy_create_tcp] **db_config - # [END_EXCLUDE] + # [START cloud_sql_postgres_sqlalchemy_create_tcp] ) # [END cloud_sql_postgres_sqlalchemy_create_tcp] + return pool + def init_unix_connection_engine(db_config): # [START cloud_sql_postgres_sqlalchemy_create_socket] - if os.environ.get("DB_SOCKET_PATH"): - socket_path = os.environ.get("DB_SOCKET_PATH") - else: - socket_path = "/cloudsql" - - return sqlalchemy.create_engine( + # Remember - storing secrets in plaintext is potentially unsafe. Consider using + # something like https://cloud.google.com/secret-manager/docs/overview to help keep + # secrets secret. + db_user = os.environ["DB_USER"] + db_pass = os.environ["DB_PASS"] + db_name = os.environ["DB_NAME"] + db_socket_dir = os.environ.get("DB_SOCKET_DIR", "/cloudsql") + cloud_sql_connection_name = os.environ["CLOUD_SQL_CONNECTION_NAME"] + + pool = sqlalchemy.create_engine( # Equivalent URL: # postgres+pg8000://:@/ # ?unix_sock=//.s.PGSQL.5432 sqlalchemy.engine.url.URL( drivername="postgres+pg8000", - username=db_user, - password=db_pass, - database=db_name, + username=db_user, # e.g. "my-database-user" + password=db_pass, # e.g. "my-database-password" + database=db_name, # e.g. "my-database-name" query={ "unix_sock": "{}/{}/.s.PGSQL.5432".format( - socket_path, - cloud_sql_connection_name) + db_socket_dir, # e.g. "/cloudsql" + cloud_sql_connection_name) # i.e "::" } ), # ... Specify additional properties here. - # [START_EXCLUDE] + # [END cloud_sql_postgres_sqlalchemy_create_socket] **db_config - # [END_EXCLUDE] + # [START cloud_sql_postgres_sqlalchemy_create_socket] ) # [END cloud_sql_postgres_sqlalchemy_create_socket] + return pool + # The SQLAlchemy engine will help manage interactions, including automatically # managing a pool of connections to your database diff --git a/cloud-sql/sql-server/sqlalchemy/README.md b/cloud-sql/sql-server/sqlalchemy/README.md index b7405ddc2cdd..d0040f83d7b9 100644 --- a/cloud-sql/sql-server/sqlalchemy/README.md +++ b/cloud-sql/sql-server/sqlalchemy/README.md @@ -31,29 +31,50 @@ [instructions](https://cloud.google.com/sql/docs/postgres/connect-external-app#4_if_required_by_your_authentication_method_create_a_service_account). Download a JSON key to use to authenticate your connection. -1. Use the information noted in the previous steps: +## Running locally +To run this application locally, download and install the `cloud_sql_proxy` by +following the instructions [here](https://cloud.google.com/sql/docs/mysql/sql-proxy#install). + +### Linux / MacOS +Use these terminal commands to initialize environment variables: ```bash export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/key.json export CLOUD_SQL_CONNECTION_NAME='::' export DB_USER='my-db-user' export DB_PASS='my-db-pass' export DB_NAME='my_db' +export DB_HOST='127.0.0.1:1433' ``` Note: Saving credentials in environment variables is convenient, but not secure - consider a more -secure solution such as [Cloud KMS](https://cloud.google.com/kms/) to help keep secrets safe. +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. -## Running locally +Then, use the following command to start the proxy in the background using TCP: +```bash +./cloud_sql_proxy -instances=${CLOUD_SQL_CONNECTION_NAME}=tcp:1433 sqlserver -u ${DB_USER} --host 127.0.0.1 & +``` -To run this application locally, download and install the `cloud_sql_proxy` by -following the instructions [here](https://cloud.google.com/sql/docs/mysql/sql-proxy#install). +### Windows / PowerShell +Use these PowerShell commands to initialize environment variables: +```powershell +$env:GOOGLE_APPLICATION_CREDENTIALS="" +$env:CLOUD_SQL_CONNECTION_NAME="::" +$env:DB_USER="my-db-user" +$env:DB_PASS="my-db-pass" +$env:DB_NAME="my_db" +$env:DB_HOST="127.0.0.1:1433" +``` +Note: Saving credentials in environment variables is convenient, but not secure - consider a more +secure solution such as [Secret Manager](https://cloud.google.com/secret-manager/docs/overview) to +help keep secrets safe. -Then, use the following command to start the proxy in the -background using TCP: -```bash -./cloud_sql_proxy -instances=${CLOUD_SQL_CONNECTION_NAME}=tcp:1433 sqlserver -u ${DB_USER} --host 127.0.0.1 +Then use this command to launch the proxy in a separate PowerShell session: +```powershell +Start-Process -filepath "C:\" -ArgumentList "-instances=::=tcp:1433 -credential_file=" ``` -Next, setup install the requirements into a virtual enviroment: +### Testing the application +Next, setup a virtual environment and install the application's requirements: ```bash virtualenv --python python3 env source env/bin/activate diff --git a/cloud-sql/sql-server/sqlalchemy/app.yaml b/cloud-sql/sql-server/sqlalchemy/app.yaml index eded261e16b1..f64d44cb088e 100644 --- a/cloud-sql/sql-server/sqlalchemy/app.yaml +++ b/cloud-sql/sql-server/sqlalchemy/app.yaml @@ -21,8 +21,7 @@ env_variables: DB_USER: DB_PASS: DB_NAME: - # Whether the app is deployed or running locally - DEPLOYED: true + DB_HOST: 172.17.0.1: beta_settings: - cloud_sql_instances: ::=tcp:1433 + cloud_sql_instances: ::=tcp: diff --git a/cloud-sql/sql-server/sqlalchemy/main.py b/cloud-sql/sql-server/sqlalchemy/main.py index a9b39f3405f5..74c814229e8c 100644 --- a/cloud-sql/sql-server/sqlalchemy/main.py +++ b/cloud-sql/sql-server/sqlalchemy/main.py @@ -29,60 +29,70 @@ logger = logging.getLogger() -# [START cloud_sql_server_sqlalchemy_create] -# Remember - storing secrets in plaintext is potentially unsafe. Consider using -# something like https://cloud.google.com/kms/ to help keep secrets secret. -db_user = os.environ.get("DB_USER") -db_pass = os.environ.get("DB_PASS") -db_name = os.environ.get("DB_NAME") - -# When deployed to GAE Flex for TCP, use "172.17.0.1" to connect -host = "172.17.0.1" if os.environ.get("DEPLOYED") else "127.0.0.1" - -# The SQLAlchemy engine will help manage interactions, including automatically -# managing a pool of connections to your database -db = sqlalchemy.create_engine( - # Equivalent URL: - # mssql+pyodbc://:@/:/?driver=ODBC+Driver+17+for+SQL+Server - sqlalchemy.engine.url.URL( - "mssql+pyodbc", - username=db_user, - password=db_pass, - database=db_name, - host=host, - port=1433, - query={"driver": "ODBC Driver 17 for SQL Server"}, - ), - # ... Specify additional properties here. - # [START_EXCLUDE] - # [START cloud_sql_server_sqlalchemy_limit] - # Pool size is the maximum number of permanent connections to keep. - pool_size=5, - # Temporarily exceeds the set pool_size if no connections are available. - max_overflow=2, - # The total number of concurrent connections for your application will be - # a total of pool_size and max_overflow. - # [END cloud_sql_server_sqlalchemy_limit] - # [START cloud_sql_server_sqlalchemy_backoff] - # SQLAlchemy automatically uses delays between failed connection attempts, - # but provides no arguments for configuration. - # [END cloud_sql_server_sqlalchemy_backoff] - # [START cloud_sql_server_sqlalchemy_timeout] - # 'pool_timeout' is the maximum number of seconds to wait when retrieving a - # new connection from the pool. After the specified amount of time, an - # exception will be thrown. - pool_timeout=30, # 30 seconds - # [END cloud_sql_server_sqlalchemy_timeout] - # [START cloud_sql_server_sqlalchemy_lifetime] - # 'pool_recycle' is the maximum number of seconds a connection can persist. - # Connections that live longer than the specified amount of time will be - # reestablished - pool_recycle=1800, # 30 minutes - # [END cloud_sql_server_sqlalchemy_lifetime] - echo=True # debug - # [END_EXCLUDE] -) -# [END cloud_sql_server_sqlalchemy_create] + +def init_tcp_connection_engine(): + # [START cloud_sql_server_sqlalchemy_create_tcp] + # Remember - storing secrets in plaintext is potentially unsafe. Consider using + # something like https://cloud.google.com/secret-manager/docs/overview to help keep + # secrets secret. + db_user = os.environ["DB_USER"] + db_pass = os.environ["DB_PASS"] + db_name = os.environ["DB_NAME"] + db_host = os.environ["DB_HOST"] + + # Extract host and port from environment variable DB_HOST + host_args = db_host.split(":") + db_hostname, db_port = host_args[0], int(host_args[1]) + + # The SQLAlchemy engine will help manage interactions, including automatically + # managing a pool of connections to your database + pool = sqlalchemy.create_engine( + # Equivalent URL: + # mssql+pyodbc://:@/:/?driver=ODBC+Driver+17+for+SQL+Server + sqlalchemy.engine.url.URL( + "mssql+pyodbc", + username=db_user, + password=db_pass, + database=db_name, + host=db_hostname, + port=db_port, + query={"driver": "ODBC Driver 17 for SQL Server"}, + ), + # ... Specify additional properties here. + # [START_EXCLUDE] + # [START cloud_sql_server_sqlalchemy_limit] + # Pool size is the maximum number of permanent connections to keep. + pool_size=5, + # Temporarily exceeds the set pool_size if no connections are available. + max_overflow=2, + # The total number of concurrent connections for your application will be + # a total of pool_size and max_overflow. + # [END cloud_sql_server_sqlalchemy_limit] + # [START cloud_sql_server_sqlalchemy_backoff] + # SQLAlchemy automatically uses delays between failed connection attempts, + # but provides no arguments for configuration. + # [END cloud_sql_server_sqlalchemy_backoff] + # [START cloud_sql_server_sqlalchemy_timeout] + # 'pool_timeout' is the maximum number of seconds to wait when retrieving a + # new connection from the pool. After the specified amount of time, an + # exception will be thrown. + pool_timeout=30, # 30 seconds + # [END cloud_sql_server_sqlalchemy_timeout] + # [START cloud_sql_server_sqlalchemy_lifetime] + # 'pool_recycle' is the maximum number of seconds a connection can persist. + # Connections that live longer than the specified amount of time will be + # reestablished + pool_recycle=1800, # 30 minutes + # [END cloud_sql_server_sqlalchemy_lifetime] + echo=True # debug + # [END_EXCLUDE] + ) + # [END cloud_sql_server_sqlalchemy_create_tcp] + + return pool + + +db = init_tcp_connection_engine() @app.before_first_request diff --git a/datastore/cloud-client/snippets.py b/datastore/cloud-client/snippets.py index 97ff5d41cd91..198d0257595e 100644 --- a/datastore/cloud-client/snippets.py +++ b/datastore/cloud-client/snippets.py @@ -784,7 +784,8 @@ def property_by_kind_run_query(client): def eventual_consistent_query(client): # [START datastore_eventual_consistent_query] - # Read consistency cannot be specified in google-cloud-python. + query = client.query(kind='Task') + query.fetch(eventual=True) # [END datastore_eventual_consistent_query] pass diff --git a/dlp/README.rst b/dlp/README.rst index 76bd9dd8dfc6..9ef0fc3fa147 100644 --- a/dlp/README.rst +++ b/dlp/README.rst @@ -339,13 +339,12 @@ To run this sample: .. code-block:: bash $ python deid.py - - usage: deid.py [-h] {deid_mask,deid_fpe,reid_fpe,deid_date_shift} ... + usage: deid.py [-h] {deid_mask,deid_fpe,reid_fpe,deid_date_shift,redact} ... Uses of the Data Loss Prevention API for deidentifying sensitive data. positional arguments: - {deid_mask,deid_fpe,reid_fpe,deid_date_shift} + {deid_mask,deid_fpe,reid_fpe,deid_date_shift,redact} Select how to submit content to the API. deid_mask Deidentify sensitive data in a string by masking it with a character. @@ -355,6 +354,8 @@ To run this sample: Preserving Encryption (FPE). deid_date_shift Deidentify dates in a CSV file by pseudorandomly shifting them. + redact Redact sensitive data in a string by replacing it with + the info type of the data. optional arguments: -h, --help show this help message and exit @@ -378,4 +379,4 @@ to `browse the source`_ and `report issues`_. https://github.com/GoogleCloudPlatform/google-cloud-python/issues -.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file +.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/dlp/deid.py b/dlp/deid.py index b08a341dd82e..81847690866c 100644 --- a/dlp/deid.py +++ b/dlp/deid.py @@ -435,6 +435,61 @@ def write_data(data): # [END dlp_deidentify_date_shift] +# [START dlp_redact_sensitive_data] +def redact_sensitive_data(project, item, info_types): + """Uses the Data Loss Prevention API to redact sensitive data in a + string by replacing it with the info type. + Args: + project: The Google Cloud project id to use as a parent resource. + item: The string to redact (will be treated as text). + info_types: A list of strings representing info types to look for. + A full list of info type categories can be fetched from the API. + Returns: + None; the response from the API is printed to the terminal. + """ + + # Import the client library + import google.cloud.dlp + + # Instantiate a client + dlp = google.cloud.dlp_v2.DlpServiceClient() + + # Convert the project id into a full resource id. + parent = dlp.project_path(project) + + # Construct inspect configuration dictionary + inspect_config = { + "info_types": [{"name": info_type} for info_type in info_types] + } + + # Construct deidentify configuration dictionary + deidentify_config = { + "info_type_transformations": { + "transformations": [ + { + "primitive_transformation": { + "replace_with_info_type_config": {} + } + } + ] + } + } + + # Call the API + response = dlp.deidentify_content( + parent, + inspect_config=inspect_config, + deidentify_config=deidentify_config, + item={"value": item}, + ) + + # Print out the results. + print(response.item.value) + + +# [END dlp_redact_sensitive_data] + + if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers( @@ -626,6 +681,30 @@ def write_data(data): "key_name.", ) + redact_parser = subparsers.add_parser( + "redact", + help="Redact sensitive data in a string by replacing it with the " + "info type of the data.", + ) + redact_parser.add_argument( + "--info_types", + action="append", + help="Strings representing info types to look for. A full list of " + "info categories and types is available from the API. Examples " + 'include "FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS". ' + "If unspecified, the three above examples will be used.", + default=["FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS"], + ) + redact_parser.add_argument( + "project", + help="The Google Cloud project id to use as a parent resource.", + ) + redact_parser.add_argument( + "item", + help="The string to redact." + "Example: 'My credit card is 4242 4242 4242 4242'", + ) + args = parser.parse_args() if args.content == "deid_mask": @@ -667,3 +746,9 @@ def write_data(data): wrapped_key=args.wrapped_key, key_name=args.key_name, ) + elif args.content == "redact": + redact_sensitive_data( + args.project, + item=args.item, + info_types=args.info_types, + ) diff --git a/dlp/deid_test.py b/dlp/deid_test.py index db14b5758e96..db0c94e35dd6 100644 --- a/dlp/deid_test.py +++ b/dlp/deid_test.py @@ -185,3 +185,17 @@ def test_reidentify_with_fpe(capsys): out, _ = capsys.readouterr() assert "731997681" not in out + + +def test_redact_sensitive_data(capsys): + url_to_redact = "https://cloud.google.com" + deid.redact_sensitive_data( + GCLOUD_PROJECT, + "My favorite site is " + url_to_redact, + ["URL"], + ) + + out, _ = capsys.readouterr() + + assert url_to_redact not in out + assert "My favorite site is [URL]" in out diff --git a/firestore/cloud-client/snippets.py b/firestore/cloud-client/snippets.py index c1a51d422034..b473a69ecafe 100644 --- a/firestore/cloud-client/snippets.py +++ b/firestore/cloud-client/snippets.py @@ -61,7 +61,7 @@ def quickstart_get_collection(): docs = users_ref.stream() for doc in docs: - print(u'{} => {}'.format(doc.id, doc.to_dict())) + print(f'{doc.id} => {doc.to_dict()}') # [END quickstart_get_collection] @@ -149,9 +149,14 @@ def to_dict(self): def __repr__(self): return( - u'City(name={}, country={}, population={}, capital={}, regions={})' - .format(self.name, self.country, self.population, self.capital, - self.regions)) + f'City(\ + name={self.name}, \ + country={self.country}, \ + population={self.population}, \ + capital={self.capital}, \ + regions={self.regions}\ + )' + ) # [END custom_class_def] @@ -219,7 +224,7 @@ def get_check_exists(): doc = doc_ref.get() if doc.exists: - print(u'Document data: {}'.format(doc.to_dict())) + print(f'Document data: {doc.to_dict()}') else: print(u'No such document!') # [END get_check_exists] @@ -239,10 +244,11 @@ def get_custom_class(): def get_simple_query(): db = firestore.Client() # [START get_simple_query] + # Note: Use of CollectionRef stream() is prefered to get() docs = db.collection(u'cities').where(u'capital', u'==', True).stream() for doc in docs: - print(u'{} => {}'.format(doc.id, doc.to_dict())) + print(f'{doc.id} => {doc.to_dict()}') # [END get_simple_query] @@ -255,7 +261,7 @@ def array_contains_filter(): # [END fs_array_contains_filter] docs = query.stream() for doc in docs: - print(u'{} => {}'.format(doc.id, doc.to_dict())) + print(f'{doc.id} => {doc.to_dict()}') def get_full_collection(): @@ -264,7 +270,7 @@ def get_full_collection(): docs = db.collection(u'cities').stream() for doc in docs: - print(u'{} => {}'.format(doc.id, doc.to_dict())) + print(f'{doc.id} => {doc.to_dict()}') # [END get_full_collection] @@ -332,7 +338,7 @@ def update_doc_array(): city_ref.update({u'regions': firestore.ArrayRemove([u'east_coast'])}) # [END fs_update_doc_array] city = city_ref.get() - print(u'Updated the regions field of the DC. {}'.format(city.to_dict())) + print(f'Updated the regions field of the DC. {city.to_dict()}') def update_multiple(): @@ -622,7 +628,7 @@ def snapshot_cursors(): # [END fs_start_at_snapshot_query_cursor] results = start_at_snapshot.limit(10).stream() for doc in results: - print(u'{}'.format(doc.id)) + print(f'{doc.id}') return results @@ -667,7 +673,7 @@ def listen_document(): # Create a callback on_snapshot function to capture changes def on_snapshot(doc_snapshot, changes, read_time): for doc in doc_snapshot: - print(u'Received document snapshot: {}'.format(doc.id)) + print(f'Received document snapshot: {doc.id}') callback_done.set() doc_ref = db.collection(u'cities').document(u'SF') @@ -705,7 +711,7 @@ def on_snapshot(col_snapshot, changes, read_time): print(u'Callback received query snapshot.') print(u'Current cities in California:') for doc in col_snapshot: - print(u'{}'.format(doc.id)) + print(f'{doc.id}') callback_done.set() col_query = db.collection(u'cities').where(u'state', u'==', u'CA') @@ -741,11 +747,11 @@ def on_snapshot(col_snapshot, changes, read_time): print(u'Current cities in California: ') for change in changes: if change.type.name == 'ADDED': - print(u'New city: {}'.format(change.document.id)) + print(f'New city: {change.document.id}') elif change.type.name == 'MODIFIED': - print(u'Modified city: {}'.format(change.document.id)) + print(f'Modified city: {change.document.id}') elif change.type.name == 'REMOVED': - print(u'Removed city: {}'.format(change.document.id)) + print(f'Removed city: {change.document.id}') delete_done.set() col_query = db.collection(u'cities').where(u'state', u'==', u'CA') @@ -835,7 +841,7 @@ def delete_collection(coll_ref, batch_size): deleted = 0 for doc in docs: - print(u'Deleting doc {} => {}'.format(doc.id, doc.to_dict())) + print(f'Deleting doc {doc.id} => {doc.to_dict()}') doc.reference.delete() deleted = deleted + 1 @@ -905,7 +911,7 @@ def collection_group_query(db): .where(u'type', u'==', u'museum') docs = museums.stream() for doc in docs: - print(u'{} => {}'.format(doc.id, doc.to_dict())) + print(f'{doc.id} => {doc.to_dict()}') # [END fs_collection_group_query] return docs diff --git a/tables/automl/automl_tables_set_endpoint.py b/tables/automl/automl_tables_set_endpoint.py new file mode 100644 index 000000000000..d6ab898b4f5d --- /dev/null +++ b/tables/automl/automl_tables_set_endpoint.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_client_with_endpoint(gcp_project_id): + """Create a Tables client with a non-default endpoint.""" + # [START automl_set_endpoint] + from google.cloud import automl_v1beta1 as automl + from google.api_core.client_options import ClientOptions + + # Set the endpoint you want to use via the ClientOptions. + # gcp_project_id = 'YOUR_PROJECT_ID' + client_options = ClientOptions(api_endpoint="eu-automl.googleapis.com:443") + client = automl.TablesClient( + project=gcp_project_id, region="eu", client_options=client_options + ) + # [END automl_set_endpoint] + + # do simple test to check client connectivity + print(client.list_datasets()) + + return client diff --git a/tables/automl/endpoint_test.py b/tables/automl/endpoint_test.py new file mode 100644 index 000000000000..b556a4c9827c --- /dev/null +++ b/tables/automl/endpoint_test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import automl_tables_set_endpoint + +PROJECT = os.environ["GCLOUD_PROJECT"] + + +def test_client_creation(capsys): + automl_tables_set_endpoint.create_client_with_endpoint(PROJECT) + out, _ = capsys.readouterr() + assert "GRPCIterator" in out