diff --git a/pom.xml b/pom.xml index 1bd18678fe89c..3be9e470bc4fd 100644 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,7 @@ 0.38 0.6 1.12.560 - 3.9.0 + 4.12.0 3.4.0 19.3.0.0 1.40 @@ -105,6 +105,9 @@ 1.68.0 -missing + 1.38.0 + 1.38.0-alpha + 1.9.10 @@ -2329,19 +2332,19 @@ io.opentelemetry opentelemetry-api - 1.19.0 + ${opentelemetry.version} io.opentelemetry opentelemetry-context - 1.19.0 + ${opentelemetry.version} io.opentelemetry opentelemetry-exporter-otlp - 1.19.0 + ${opentelemetry.version} com.squareup.okhttp3 @@ -2353,31 +2356,37 @@ io.opentelemetry opentelemetry-extension-trace-propagators - 1.19.0 + ${opentelemetry.version} io.opentelemetry opentelemetry-sdk - 1.19.0 + ${opentelemetry.version} + + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} io.opentelemetry opentelemetry-sdk-common - 1.19.0 + ${opentelemetry.version} io.opentelemetry opentelemetry-sdk-trace - 1.19.0 + ${opentelemetry.version} io.opentelemetry opentelemetry-semconv - 1.19.0-alpha + ${opentelemetry-alpha.version} @@ -2475,7 +2484,12 @@ flight-core ${dep.arrow.version} - + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin-stdlib-jdk8.version} + diff --git a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java index 08e3b79b6b3a6..16c2f02905c01 100644 --- a/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java +++ b/presto-benchmark-driver/src/main/java/com/facebook/presto/benchmark/driver/BenchmarkDriverOptions.java @@ -94,7 +94,6 @@ public ClientSession getClientSession() parseServer(server), user, "presto-benchmark", - Optional.empty(), ImmutableSet.of(), null, catalog, diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java index 3fe4f25c6cca0..799335ab43317 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java @@ -82,7 +82,6 @@ public class TestCassandraConnector "user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), - Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), diff --git a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java index 6a05483e1e511..a2edcafbe16e4 100644 --- a/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java +++ b/presto-cli/src/main/java/com/facebook/presto/cli/ClientOptions.java @@ -180,7 +180,6 @@ public ClientSession toClientSession() parseServer(server), user, source, - Optional.empty(), parseClientTags(clientTags), clientInfo, catalog, diff --git a/presto-cli/src/test/java/com/facebook/presto/cli/AbstractCliTest.java b/presto-cli/src/test/java/com/facebook/presto/cli/AbstractCliTest.java index eb2267e1bd797..8d74528484da3 100644 --- a/presto-cli/src/test/java/com/facebook/presto/cli/AbstractCliTest.java +++ b/presto-cli/src/test/java/com/facebook/presto/cli/AbstractCliTest.java @@ -68,7 +68,6 @@ protected ClientSession createMockClientSession() server.url("/").uri(), "user", "source", - Optional.empty(), ImmutableSet.of(), "clientInfo", "catalog", diff --git a/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java b/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java index 5896a3c6f2d3c..994dea03217f8 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java +++ b/presto-client/src/main/java/com/facebook/presto/client/ClientSession.java @@ -24,7 +24,6 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Set; import static com.google.common.base.MoreObjects.toStringHelper; @@ -37,7 +36,6 @@ public class ClientSession private final URI server; private final String user; private final String source; - private final Optional traceToken; private final Set clientTags; private final String clientInfo; private final String catalog; @@ -72,7 +70,6 @@ public ClientSession( URI server, String user, String source, - Optional traceToken, Set clientTags, String clientInfo, String catalog, @@ -94,7 +91,6 @@ public ClientSession( this.server = requireNonNull(server, "server is null"); this.user = user; this.source = source; - this.traceToken = requireNonNull(traceToken, "traceToken is null"); this.clientTags = ImmutableSet.copyOf(requireNonNull(clientTags, "clientTags is null")); this.clientInfo = clientInfo; this.catalog = catalog; @@ -165,11 +161,6 @@ public String getSource() return source; } - public Optional getTraceToken() - { - return traceToken; - } - public Set getClientTags() { return clientTags; @@ -273,7 +264,6 @@ public String toString() .add("clientInfo", clientInfo) .add("catalog", catalog) .add("schema", schema) - .add("traceToken", traceToken.orElse(null)) .add("timeZone", timeZone) .add("locale", locale) .add("properties", properties) @@ -287,7 +277,6 @@ public static final class Builder private URI server; private String user; private String source; - private Optional traceToken; private Set clientTags; private String clientInfo; private String catalog; @@ -312,7 +301,6 @@ private Builder(ClientSession clientSession) server = clientSession.getServer(); user = clientSession.getUser(); source = clientSession.getSource(); - traceToken = clientSession.getTraceToken(); clientTags = clientSession.getClientTags(); clientInfo = clientSession.getClientInfo(); catalog = clientSession.getCatalog(); @@ -410,7 +398,6 @@ public ClientSession build() server, user, source, - traceToken, clientTags, clientInfo, catalog, diff --git a/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java b/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java index 50f7d202cad7b..73039a29a4e16 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java +++ b/presto-client/src/main/java/com/facebook/presto/client/OkHttpUtil.java @@ -14,6 +14,7 @@ package com.facebook.presto.client; import com.facebook.airlift.security.pem.PemReader; +import com.facebook.presto.client.okhttp3.internal.tls.LegacyHostnameVerifier; import com.google.common.base.CharMatcher; import com.google.common.net.HostAndPort; import okhttp3.Call; @@ -237,6 +238,7 @@ public static void setupSsl( sslContext.init(keyManagers, new TrustManager[] {trustManager}, null); clientBuilder.sslSocketFactory(sslContext.getSocketFactory(), trustManager); + clientBuilder.hostnameVerifier(LegacyHostnameVerifier.INSTANCE); } catch (GeneralSecurityException | IOException e) { throw new ClientException("Error setting up SSL: " + e.getMessage(), e); diff --git a/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java b/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java index 3be0b241ee0b2..9f83194cad39f 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java +++ b/presto-client/src/main/java/com/facebook/presto/client/PrestoHeaders.java @@ -21,7 +21,6 @@ public final class PrestoHeaders public static final String PRESTO_SCHEMA = "X-Presto-Schema"; public static final String PRESTO_TIME_ZONE = "X-Presto-Time-Zone"; public static final String PRESTO_LANGUAGE = "X-Presto-Language"; - public static final String PRESTO_TRACE_TOKEN = "X-Presto-Trace-Token"; public static final String PRESTO_SESSION = "X-Presto-Session"; public static final String PRESTO_SET_CATALOG = "X-Presto-Set-Catalog"; public static final String PRESTO_SET_SCHEMA = "X-Presto-Set-Schema"; diff --git a/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java b/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java index 9268195710af3..67c64171634d6 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java +++ b/presto-client/src/main/java/com/facebook/presto/client/StatementClientV1.java @@ -69,7 +69,6 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_SOURCE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_STARTED_TRANSACTION_ID; import static com.facebook.presto.client.PrestoHeaders.PRESTO_TIME_ZONE; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_TRACE_TOKEN; import static com.facebook.presto.client.PrestoHeaders.PRESTO_TRANSACTION_ID; import static com.facebook.presto.client.PrestoHeaders.PRESTO_USER; import static com.google.common.base.MoreObjects.firstNonNull; @@ -163,8 +162,6 @@ private Request buildQueryRequest(ClientSession session, String query) builder.addHeader(PRESTO_SOURCE, session.getSource()); } - session.getTraceToken().ifPresent(token -> builder.addHeader(PRESTO_TRACE_TOKEN, token)); - if (session.getClientTags() != null && !session.getClientTags().isEmpty()) { builder.addHeader(PRESTO_CLIENT_TAGS, Joiner.on(",").join(session.getClientTags())); } diff --git a/presto-client/src/main/java/com/facebook/presto/client/okhttp3/internal/tls/DistinguishedNameParser.java b/presto-client/src/main/java/com/facebook/presto/client/okhttp3/internal/tls/DistinguishedNameParser.java new file mode 100644 index 0000000000000..65bd1faf03840 --- /dev/null +++ b/presto-client/src/main/java/com/facebook/presto/client/okhttp3/internal/tls/DistinguishedNameParser.java @@ -0,0 +1,422 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.client.okhttp3.internal.tls; + +import javax.security.auth.x500.X500Principal; + +/** + * A distinguished name (DN) parser. This parser only supports extracting a string value from a DN. + * It doesn't support values in the hex-string style. + */ +final class DistinguishedNameParser +{ + private final String dn; + private final int length; + private int pos; + private int beg; + private int end; + + /** + * Temporary variable to store positions of the currently parsed item. + */ + private int cur; + + /** + * Distinguished name characters. + */ + private char[] chars; + + DistinguishedNameParser(X500Principal principal) + { + // RFC2253 is used to ensure we get attributes in the reverse + // order of the underlying ASN.1 encoding, so that the most + // significant values of repeated attributes occur first. + this.dn = principal.getName(X500Principal.RFC2253); + this.length = this.dn.length(); + } + + // gets next attribute type: (ALPHA 1*keychar) / oid + private String nextAT() + { + for (; pos < length && chars[pos] == ' '; pos++) { + // skip preceding space chars, they can present after + // comma or semicolon (compatibility with RFC 1779) + } + if (pos == length) { + return null; // reached the end of DN + } + + // mark the beginning of attribute type + beg = pos; + + // attribute type chars + pos++; + for (; pos < length && chars[pos] != '=' && chars[pos] != ' '; pos++) { + // we don't follow exact BNF syntax here: + // accept any char except space and '=' + } + if (pos >= length) { + throw new IllegalStateException("Unexpected end of DN: " + dn); + } + + // mark the end of attribute type + end = pos; + + if (chars[pos] == ' ') { + for (; pos < length && chars[pos] != '=' && chars[pos] == ' '; pos++) { + // skip trailing space chars between attribute type and '=' + // (compatibility with RFC 1779) + } + + if (chars[pos] != '=' || pos == length) { + throw new IllegalStateException("Unexpected end of DN: " + dn); + } + } + + pos++; //skip '=' char + + for (; pos < length && chars[pos] == ' '; pos++) { + // skip space chars between '=' and attribute value + // (compatibility with RFC 1779) + } + + // in case of oid attribute type skip its prefix: "oid." or "OID." + // (compatibility with RFC 1779) + if ((end - beg > 4) && (chars[beg + 3] == '.') + && (chars[beg] == 'O' || chars[beg] == 'o') + && (chars[beg + 1] == 'I' || chars[beg + 1] == 'i') + && (chars[beg + 2] == 'D' || chars[beg + 2] == 'd')) { + beg += 4; + } + + return new String(chars, beg, end - beg); + } + + // gets quoted attribute value: QUOTATION *( quotechar / pair ) QUOTATION + private String quotedAV() + { + pos++; + beg = pos; + end = beg; + while (true) { + if (pos == length) { + throw new IllegalStateException("Unexpected end of DN: " + dn); + } + + if (chars[pos] == '"') { + // enclosing quotation was found + pos++; + break; + } + else if (chars[pos] == '\\') { + chars[end] = getEscaped(); + } + else { + // shift char: required for string with escaped chars + chars[end] = chars[pos]; + } + pos++; + end++; + } + + for (; pos < length && chars[pos] == ' '; pos++) { + // skip trailing space chars before comma or semicolon. + // (compatibility with RFC 1779) + } + + return new String(chars, beg, end - beg); + } + + // gets hex string attribute value: "#" hexstring + private String hexAV() + { + if (pos + 4 >= length) { + // encoded byte array must be not less then 4 c + throw new IllegalStateException("Unexpected end of DN: " + dn); + } + + beg = pos; // store '#' position + pos++; + while (true) { + // check for end of attribute value + // looks for space and component separators + if (pos == length || chars[pos] == '+' || chars[pos] == ',' + || chars[pos] == ';') { + end = pos; + break; + } + + if (chars[pos] == ' ') { + end = pos; + pos++; + for (; pos < length && chars[pos] == ' '; pos++) { + // skip trailing space chars before comma or semicolon. + // (compatibility with RFC 1779) + } + break; + } + else if (chars[pos] >= 'A' && chars[pos] <= 'F') { + chars[pos] += 32; //to low case + } + + pos++; + } + + // verify length of hex string + // encoded byte array must be not less then 4 and must be even number + int hexLen = end - beg; // skip first '#' char + if (hexLen < 5 || (hexLen & 1) == 0) { + throw new IllegalStateException("Unexpected end of DN: " + dn); + } + + // get byte encoding from string representation + byte[] encoded = new byte[hexLen / 2]; + for (int i = 0, p = beg + 1; i < encoded.length; p += 2, i++) { + encoded[i] = (byte) getByte(p); + } + return new String(chars, beg, hexLen); + } + // gets string attribute value: *( stringchar / pair) + private String escapedAV() + { + beg = pos; + end = pos; + while (true) { + if (pos >= length) { + // the end of DN has been found + return new String(chars, beg, end - beg); + } + + switch (chars[pos]) { + case '+': + case ',': + case ';': + // separator char has been found + return new String(chars, beg, end - beg); + case '\\': + // escaped char + chars[end++] = getEscaped(); + pos++; + break; + case ' ': + // need to figure out whether space defines + // the end of attribute value or not + cur = end; + + pos++; + chars[end++] = ' '; + + for (; pos < length && chars[pos] == ' '; pos++) { + chars[end++] = ' '; + } + if (pos == length || chars[pos] == ',' || chars[pos] == '+' + || chars[pos] == ';') { + // separator char or the end of DN has been found + return new String(chars, beg, cur - beg); + } + break; + default: + chars[end++] = chars[pos]; + pos++; + } + } + } + + // returns escaped char + private char getEscaped() + { + pos++; + if (pos == length) { + throw new IllegalStateException("Unexpected end of DN: " + dn); + } + + switch (chars[pos]) { + case '"': + case '\\': + case ',': + case '=': + case '+': + case '<': + case '>': + case '#': + case ';': + case ' ': + case '*': + case '%': + case '_': + //FIXME: escaping is allowed only for leading or trailing space char + return chars[pos]; + default: + // RFC doesn't explicitly say that escaped hex pair is + // interpreted as UTF-8 char. It only contains an example of such DN. + return getUTF8(); + } + } + + // decodes UTF-8 char + // see http://www.unicode.org for UTF-8 bit distribution table + private char getUTF8() + { + int res = getByte(pos); + pos++; //FIXME tmp + + if (res < 128) { // one byte: 0-7F + return (char) res; + } + else if (res >= 192 && res <= 247) { + int count; + if (res <= 223) { // two bytes: C0-DF + count = 1; + res = res & 0x1F; + } + else if (res <= 239) { // three bytes: E0-EF + count = 2; + res = res & 0x0F; + } + else { // four bytes: F0-F7 + count = 3; + res = res & 0x07; + } + int b; + for (int i = 0; i < count; i++) { + pos++; + if (pos == length || chars[pos] != '\\') { + return 0x3F; //FIXME failed to decode UTF-8 char - return '?' + } + pos++; + + b = getByte(pos); + pos++; //FIXME tmp + if ((b & 0xC0) != 0x80) { + return 0x3F; //FIXME failed to decode UTF-8 char - return '?' + } + + res = (res << 6) + (b & 0x3F); + } + return (char) res; + } + else { + return 0x3F; //FIXME failed to decode UTF-8 char - return '?' + } + } + + // Returns byte representation of a char pair + // The char pair is composed of DN char in + // specified 'position' and the next char + // According to BNF syntax: + // hexchar = DIGIT / "A" / "B" / "C" / "D" / "E" / "F" + // / "a" / "b" / "c" / "d" / "e" / "f" + private int getByte(int position) + { + if (position + 1 >= length) { + throw new IllegalStateException("Malformed DN: " + dn); + } + + int b1; + int b2; + + b1 = chars[position]; + if (b1 >= '0' && b1 <= '9') { + b1 = b1 - '0'; + } + else if (b1 >= 'a' && b1 <= 'f') { + b1 = b1 - 87; // 87 = 'a' - 10 + } + else if (b1 >= 'A' && b1 <= 'F') { + b1 = b1 - 55; // 55 = 'A' - 10 + } + else { + throw new IllegalStateException("Malformed DN: " + dn); + } + + b2 = chars[position + 1]; + if (b2 >= '0' && b2 <= '9') { + b2 = b2 - '0'; + } + else if (b2 >= 'a' && b2 <= 'f') { + b2 = b2 - 87; // 87 = 'a' - 10 + } + else if (b2 >= 'A' && b2 <= 'F') { + b2 = b2 - 55; // 55 = 'A' - 10 + } + else { + throw new IllegalStateException("Malformed DN: " + dn); + } + return (b1 << 4) + b2; + } + + /** + * Parses the DN and returns the most significant attribute value for an attribute type, or null + * if none found. + * + * @param attributeType attribute type to look for (e.g. "ca") + */ + public String findMostSpecific(String attributeType) + { + // Initialize internal state. + pos = 0; + beg = 0; + end = 0; + cur = 0; + chars = dn.toCharArray(); + + String attType = nextAT(); + if (attType == null) { + return null; + } + while (true) { + String attValue = ""; + + if (pos == length) { + return null; + } + switch (chars[pos]) { + case '"': + attValue = quotedAV(); + break; + case '#': + attValue = hexAV(); + break; + case '+': + case ',': + case ';': // compatibility with RFC 1779: semicolon can separate RDNs + //empty attribute value + break; + default: + attValue = escapedAV(); + } + + // Values are ordered from most specific to least specific + // due to the RFC2253 formatting. So take the first match + // we see. + if (attributeType.equalsIgnoreCase(attType)) { + return attValue; + } + + if (pos >= length) { + return null; + } + if (chars[pos] == ',' || chars[pos] == ';') { + //Do nothing + } else if (chars[pos] != '+') { + throw new IllegalStateException("Malformed DN: " + dn); + } + pos++; + attType = nextAT(); + if (attType == null) { + throw new IllegalStateException("Malformed DN: " + dn); + } + } + } +} diff --git a/presto-client/src/main/java/com/facebook/presto/client/okhttp3/internal/tls/LegacyHostnameVerifier.java b/presto-client/src/main/java/com/facebook/presto/client/okhttp3/internal/tls/LegacyHostnameVerifier.java new file mode 100644 index 0000000000000..79aa87506ec89 --- /dev/null +++ b/presto-client/src/main/java/com/facebook/presto/client/okhttp3/internal/tls/LegacyHostnameVerifier.java @@ -0,0 +1,262 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.client.okhttp3.internal.tls; + +import okhttp3.internal.tls.OkHostnameVerifier; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLSession; +import javax.security.auth.x500.X500Principal; + +import java.security.cert.Certificate; +import java.security.cert.CertificateParsingException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +public class LegacyHostnameVerifier + implements HostnameVerifier +{ + private static final int ALT_DNS_NAME = 2; + private static final int ALT_IPA_NAME = 7; + private static final Pattern VERIFY_AS_IP_ADDRESS = Pattern.compile( + "([0-9a-fA-F]*:[0-9a-fA-F:.]*)|([\\d.]+)"); + + public static final HostnameVerifier INSTANCE = new LegacyHostnameVerifier(); + + private LegacyHostnameVerifier() + { + } + + @Override + public boolean verify(String host, SSLSession session) + { + if (OkHostnameVerifier.INSTANCE.verify(host, session)) { + return true; + } + + // the CN cannot be used with IP addresses + if (verifyAsIpAddress(host)) { + return false; + } + + // try to verify using the legacy CN rules + try { + Certificate[] certificates = session.getPeerCertificates(); + X509Certificate certificate = (X509Certificate) certificates[0]; + + // only use CN if there are no alt names + if (!allSubjectAltNames(certificate).isEmpty()) { + return false; + } + + X500Principal principal = certificate.getSubjectX500Principal(); + // RFC 2818 advises using the most specific name for matching. + String cn = new DistinguishedNameParser(principal).findMostSpecific("cn"); + if (cn != null) { + return verifyHostName(host, cn); + } + + return false; + } + catch (SSLException e) { + return false; + } + } + + static boolean verifyAsIpAddress(String host) + { + return VERIFY_AS_IP_ADDRESS.matcher(host).matches(); + } + + /** + * Returns true if {@code certificate} matches {@code ipAddress}. + */ + private boolean verifyIpAddress(String ipAddress, X509Certificate certificate) + { + List altNames = getSubjectAltNames(certificate, ALT_IPA_NAME); + for (int i = 0, size = altNames.size(); i < size; i++) { + if (ipAddress.equalsIgnoreCase(altNames.get(i))) { + return true; + } + } + return false; + } + + private boolean verifyHostName(String hostName, X509Certificate certificate) + { + hostName = hostName.toLowerCase(Locale.US); + boolean hasDns = false; + List altNames = getSubjectAltNames(certificate, ALT_DNS_NAME); + for (int i = 0, size = altNames.size(); i < size; i++) { + hasDns = true; + if (verifyHostName(hostName, altNames.get(i))) { + return true; + } + } + + if (!hasDns) { + X500Principal principal = certificate.getSubjectX500Principal(); + // RFC 2818 advises using the most specific name for matching. + String cn = new DistinguishedNameParser(principal).findMostSpecific("cn"); + if (cn != null) { + return verifyHostName(hostName, cn); + } + } + + return false; + } + + public static List allSubjectAltNames(X509Certificate certificate) + { + List altIpaNames = getSubjectAltNames(certificate, ALT_IPA_NAME); + List altDnsNames = getSubjectAltNames(certificate, ALT_DNS_NAME); + List result = new ArrayList<>(altIpaNames.size() + altDnsNames.size()); + result.addAll(altIpaNames); + result.addAll(altDnsNames); + return result; + } + + private static List getSubjectAltNames(X509Certificate certificate, int type) + { + List result = new ArrayList<>(); + try { + Collection subjectAltNames = certificate.getSubjectAlternativeNames(); + if (subjectAltNames == null) { + return Collections.emptyList(); + } + for (Object subjectAltName : subjectAltNames) { + List entry = (List) subjectAltName; + if (entry == null || entry.size() < 2) { + continue; + } + Integer altNameType = (Integer) entry.get(0); + if (altNameType == null) { + continue; + } + if (altNameType == type) { + String altName = (String) entry.get(1); + if (altName != null) { + result.add(altName); + } + } + } + return result; + } + catch (CertificateParsingException e) { + return Collections.emptyList(); + } + } + + /** + * Returns {@code true} iff {@code hostName} matches the domain name {@code pattern}. + * + * @param hostName lower-case host name. + * @param pattern domain name pattern from certificate. May be a wildcard pattern such as + * {@code *.android.com}. + */ + private boolean verifyHostName(String hostName, String pattern) + { + // Basic sanity checks + // Check length == 0 instead of .isEmpty() to support Java 5. + if ((hostName == null) || (hostName.length() == 0) || (hostName.startsWith(".")) + || (hostName.endsWith(".."))) { + // Invalid domain name + return false; + } + if ((pattern == null) || (pattern.length() == 0) || (pattern.startsWith(".")) + || (pattern.endsWith(".."))) { + // Invalid pattern/domain name + return false; + } + + // Normalize hostName and pattern by turning them into absolute domain names if they are not + // yet absolute. This is needed because server certificates do not normally contain absolute + // names or patterns, but they should be treated as absolute. At the same time, any hostName + // presented to this method should also be treated as absolute for the purposes of matching + // to the server certificate. + // www.android.com matches www.android.com + // www.android.com matches www.android.com. + // www.android.com. matches www.android.com. + // www.android.com. matches www.android.com + if (!hostName.endsWith(".")) { + hostName += '.'; + } + if (!pattern.endsWith(".")) { + pattern += '.'; + } + // hostName and pattern are now absolute domain names. + + pattern = pattern.toLowerCase(Locale.US); + // hostName and pattern are now in lower case -- domain names are case-insensitive. + + if (!pattern.contains("*")) { + // Not a wildcard pattern -- hostName and pattern must match exactly. + return hostName.equals(pattern); + } + // Wildcard pattern + + // WILDCARD PATTERN RULES: + // 1. Asterisk (*) is only permitted in the left-most domain name label and must be the + // only character in that label (i.e., must match the whole left-most label). + // For example, *.example.com is permitted, while *a.example.com, a*.example.com, + // a*b.example.com, a.*.example.com are not permitted. + // 2. Asterisk (*) cannot match across domain name labels. + // For example, *.example.com matches test.example.com but does not match + // sub.test.example.com. + // 3. Wildcard patterns for single-label domain names are not permitted. + + if ((!pattern.startsWith("*.")) || (pattern.indexOf('*', 1) != -1)) { + // Asterisk (*) is only permitted in the left-most domain name label and must be the only + // character in that label + return false; + } + + // Optimization: check whether hostName is too short to match the pattern. hostName must be at + // least as long as the pattern because asterisk must match the whole left-most label and + // hostName starts with a non-empty label. Thus, asterisk has to match one or more characters. + if (hostName.length() < pattern.length()) { + // hostName too short to match the pattern. + return false; + } + + if ("*.".equals(pattern)) { + // Wildcard pattern for single-label domain name -- not permitted. + return false; + } + + // hostName must end with the region of pattern following the asterisk. + String suffix = pattern.substring(1); + if (!hostName.endsWith(suffix)) { + // hostName does not end with the suffix + return false; + } + + // Check that asterisk did not match across domain name labels. + int suffixStartIndexInHostName = hostName.length() - suffix.length(); + if ((suffixStartIndexInHostName > 0) + && (hostName.lastIndexOf('.', suffixStartIndexInHostName - 1) != -1)) { + // Asterisk is matching across domain name labels -- not permitted. + return false; + } + + // hostName matches pattern + return true; + } +} diff --git a/presto-common/src/main/java/com/facebook/presto/common/TelemetryConfig.java b/presto-common/src/main/java/com/facebook/presto/common/TelemetryConfig.java new file mode 100644 index 0000000000000..bab321feb769d --- /dev/null +++ b/presto-common/src/main/java/com/facebook/presto/common/TelemetryConfig.java @@ -0,0 +1,183 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.common; + +import java.util.Map; + +import static java.util.Objects.nonNull; +import static java.util.Objects.requireNonNull; + +/** + * The type TelemetryConfig to store all the values in telemetry.properties. + */ +public class TelemetryConfig +{ + private static TelemetryConfig telemetryConfig; + + private String tracingBackendUrl; + private Integer maxExporterBatchSize; + private Integer maxQueueSize; + private Integer exporterTimeout; + private Integer scheduleDelay; + private Double samplingRatio; + private Boolean tracingEnabled = false; + private Boolean spanSampling = false; + + /** + * The type Telemetry config constants. + */ + public static class TelemetryConfigConstants + { + private static final String TRACING_ENABLED = "tracing-enabled"; + private static final String TRACING_BACKEND_URL = "tracing-backend-url"; + private static final String MAX_EXPORTER_BATCH_SIZE = "max-exporter-batch-size"; + private static final String MAX_QUEUE_SIZE = "max-queue-size"; + private static final String SCHEDULE_DELAY = "schedule-delay"; + private static final String EXPORTER_TIMEOUT = "exporter-timeout"; + private static final String TRACE_SAMPLING_RATIO = "trace-sampling-ratio"; + private static final String SPAN_SAMPLING = "span-sampling"; + } + + private TelemetryConfig() + { + } + + /** + * Gets telemetry config. + * + * @return the telemetry config + */ + public static TelemetryConfig getTelemetryConfig() + { // prevent multiple instance creation + telemetryConfig = nonNull(telemetryConfig) ? telemetryConfig : new TelemetryConfig(); + return telemetryConfig; + } + + /** + * Sets telemetry properties. + * + * @param telemetryProperties the telemetry properties + */ + public void setTelemetryProperties(Map telemetryProperties) + { + tracingBackendUrl = requireNonNull(telemetryProperties.get(TelemetryConfigConstants.TRACING_BACKEND_URL), "exporter endpoint cant be null"); + maxQueueSize = Integer.valueOf(requireNonNull(telemetryProperties.get(TelemetryConfigConstants.MAX_QUEUE_SIZE), "max queue size cant be null")); + maxExporterBatchSize = Integer.valueOf(requireNonNull(telemetryProperties.get(TelemetryConfigConstants.MAX_EXPORTER_BATCH_SIZE), "max exporter batch size cant be null")); + exporterTimeout = Integer.valueOf(requireNonNull(telemetryProperties.get(TelemetryConfigConstants.EXPORTER_TIMEOUT), "exporter timeout cant be null")); + tracingEnabled = Boolean.valueOf(requireNonNull(telemetryProperties.get(TelemetryConfigConstants.TRACING_ENABLED), "trace enabled cant be null")); + scheduleDelay = Integer.valueOf(requireNonNull(telemetryProperties.get(TelemetryConfigConstants.SCHEDULE_DELAY), "schedule delay cant be null")); + samplingRatio = Double.valueOf(requireNonNull(telemetryProperties.get(TelemetryConfigConstants.TRACE_SAMPLING_RATIO), "sampling ratio cant be null")); + spanSampling = Boolean.valueOf(requireNonNull(telemetryProperties.get(TelemetryConfigConstants.SPAN_SAMPLING), "span sampling must be provided")); + } + + /** + * Sets tracing enabled. For dynamically enable/disable from /v1/telemetry/config endpoint + * + * @param tracingEnabled the tracing enabled + */ + public void setTracingEnabled(Boolean tracingEnabled) + { + getTelemetryConfig().tracingEnabled = tracingEnabled; + } + + /** + * Sets span sampling. + * + * @param spanSampling the span sampling + */ + public void setSpanSampling(Boolean spanSampling) + { + getTelemetryConfig().spanSampling = spanSampling; + } + + /** + * Gets exporter endpoint. + * + * @return the exporter endpoint + */ + public String getTracingBackendUrl() + { + return this.tracingBackendUrl; + } + + /** + * Gets max exporter batch size. + * + * @return the max exporter batch size + */ + public Integer getMaxExporterBatchSize() + { + return this.maxExporterBatchSize; + } + + /** + * Gets max queue size. + * + * @return the max queue size + */ + public Integer getMaxQueueSize() + { + return this.maxQueueSize; + } + + /** + * Gets exporter timeout. + * + * @return the exporter timeout + */ + public Integer getExporterTimeout() + { + return this.exporterTimeout; + } + + /** + * Gets schedule delay. + * + * @return the schedule delay + */ + public Integer getScheduleDelay() + { + return this.scheduleDelay; + } + + /** + * Gets sampling ratio. + * + * @return the sampling ratio + */ + public Double getSamplingRatio() + { + return this.samplingRatio; + } + + /** + * Gets tracing enabled. + * + * @return the tracing enabled + */ + public static Boolean getTracingEnabled() + { + return getTelemetryConfig().tracingEnabled; + } + + /** + * Gets span sampling. + * + * @return the span sampling + */ + public static Boolean getSpanSampling() + { + return getTelemetryConfig().spanSampling; + } +} diff --git a/presto-common/src/main/java/com/facebook/presto/common/TracingConfig.java b/presto-common/src/main/java/com/facebook/presto/common/TracingConfig.java new file mode 100644 index 0000000000000..f05834e52adbc --- /dev/null +++ b/presto-common/src/main/java/com/facebook/presto/common/TracingConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.common; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.concurrent.Immutable; + +/** + * The type Tracing config. + */ +@Immutable +public class TracingConfig +{ + private boolean tracingEnabled; + + /** + * Instantiates a new Tracing config. + * + * @param tracingEnabled the tracing enabled + */ + @JsonCreator + public TracingConfig( + @JsonProperty("tracingEnabled") boolean tracingEnabled) + { + this.tracingEnabled = tracingEnabled; + } + + /** + * Is tracing enabled boolean. + * + * @return the boolean + */ + @JsonProperty + public boolean isTracingEnabled() + { + return tracingEnabled; + } +} diff --git a/presto-common/src/main/java/com/facebook/presto/common/telemetry/tracing/TracingEnum.java b/presto-common/src/main/java/com/facebook/presto/common/telemetry/tracing/TracingEnum.java new file mode 100644 index 0000000000000..100774a3aa32a --- /dev/null +++ b/presto-common/src/main/java/com/facebook/presto/common/telemetry/tracing/TracingEnum.java @@ -0,0 +1,206 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.common.telemetry.tracing; + +public enum TracingEnum +{ + ROOT("POST /v1/statement", ""), + QUERY("query", ""), + DISPATCH("dispatch", ""), + ANALYZER("analyzer", ""), + ANALYZE("analyze", ""), + PLANNER("planner", ""), + SCHEDULER("scheduler", ""), + QUERY_START("query_start", ""), + TASK("task", ""), + SPLIT("split", ""), + PIPELINE("pipeline", ""), + REMOTE_TASK("remote-task", ""), + LIST_SCHEMA_NAMES("listSchemaNames", ""), + GET_TABLE_HANDLE_FOR_STATISTICS_COLLECTION("getTableHandleForStatisticsCollection", ""), + GET_HANDLE_VERSION("getHandleVersion", ""), + GET_SYSTEM_TABLE("getSystemTable", ""), + GET_LAYOUT("getLayout", ""), + GET_ALTERNATE_TABLE_HANDLE("getAlternativeTableHandle", ""), + IS_LEGACY_GET_LAYOUT_SUPPORTED("isLegacyGetLayoutSupported", ""), + GET_COMMON_PARTITIONING("getCommonPartitioning", ""), + IS_REFINED_PARTITIONING_OVER("isRefinedPartitioningOver", ""), + GET_PARTITIONING_HANDLE("getPartitioningHandleForExchange", ""), + GET_PARTITIONING_HANDLE_CTE("getPartitioningHandleForCteMaterialization", ""), + GET_INFO("getInfo", ""), + CHECK_CAN_ADD_COLUMNS("checkCanAddColumns", ""), + GET_TABLE_METADATA("getTableMetadata", ""), + GET_TABLE_STATISTICS("getTableStatistics", ""), + GET_COLUMN_HANDLES("getColumnHandles", ""), + GET_COLUMN_METADATA("getColumnMetadata", ""), + EXPLAIN_IO_CONSTRAINTS("toExplainIOConstraints", ""), + LIST_TABLES("Metadata.listTables", ""), + LIST_TABLE_COLUMNS("Metadata.listTableColumns", ""), + CREATE_SCHEMA("createSchema", ""), + DROP_SCHEMA("Metadata.dropSchema", ""), + RENAME_SCHEMA("Metadata.renameSchema", ""), + CREATE_TABLE("Metadata.createTable", ""), + CREATE_TEMPORARY_TABLE("Metadata.createTemporaryTable", ""), + CREATE_CONNECTOR_PARTITIONING_METADATA("createConnectorPartitioningMetadata", ""), + RENAME_TABLE("Metadata.renameTable", ""), + RENAME_COLUMN("renameColumn", ""), + ADD_COLUMN("addColumn", ""), + DROP_COLUMN("dropColumn", ""), + SET_COLUMN_TYPE("setColumnType", ""), + DROP_TABLE("dropTable", ""), + TRUNCATE_TABLE("truncateTable", ""), + GET_INSERT_LAYOUT("getInsertLayout", ""), + GET_PREFERRED_SHUFFLED_LAYOUT_FOR_INSERT("getPreferredShuffleLayoutForInsert", ""), + GET_STATISTICS_COLLECTION_METADATA_FOR_WRITE("Metadata.getStatisticsCollectionMetadataForWrite", ""), + GET_STATISTICS_COLLECTION_METADATA("Metadata.getStatisticsCollectionMetadata", ""), + BEGIN_STATISTICS_COLLECTION("beginStatisticsCollection", ""), + FINISH_STATISTICS_COLLECTION("finishStatisticsCollection", ""), + GET_NEW_TABLE_LAYOUT("getNewTableLayout", ""), + GET_PREFERRED_SHUFFLE_LAYOUT_FOR_NEW_TABLE("getPreferredShuffleLayoutForNewTable", ""), + BEGIN_QUERY("beginQuery", ""), + REGISTER_CATALOG_FOR_QUERY_ID("registerCatalogForQueryId", ""), + CLEAN_UP_QUERY("cleanupQuery", ""), + BEGIN_CREATE_TABLE("Metadata.beginCreateTable", ""), + FINISH_CREATE_TABLE("finishCreateTable", ""), + BEGIN_INSERT("beginInsert", ""), + FINISH_INSERT("finishInsert", ""), + GET_DELETE_ROW_ID_COLUMN_HANDLE("getDeleteRowIdColumnHandle", ""), + GET_UPDATE_ROW_ID_COLUMN_HANDLE("getUpdateRowIdColumnHandle", ""), + SUPPORTS_METADATA_DELETE("supportsMetadataDelete", ""), + METADATA_DELETE("metadataDelete", ""), + BEGIN_DELETE("beginDelete", ""), + FINISH_DELETE("finishDelete", ""), + BEGIN_UPDATE("beginUpdate", ""), + FINISH_UPDATE("finishUpdate", ""), + GET_CATALOG_HANDLE("Metadata.getCatalogHandle", ""), + GET_CATALOG_NAMES("getCatalogNames", ""), + LIST_VIEWS("Metadata.listViews", ""), + GET_VIEWS("Metadata.getViews", ""), + CREATE_VIEW("Metadata.createView", ""), + DROP_VIEW("Metadata.dropView", ""), + RENAME_VIEW("renameView", ""), + CREATE_MATERIALIZED_VIEW("Metadata.createMaterializedView", ""), + DROP_MATERIALIZED_VIEW("Metadata.dropMaterializedView", ""), + GET_MATERIALIZED_VIEW_STATUS("Metadata.getMaterializedViewStatus", ""), + BEGIN_REFRESH_MATERIALIZED_VIEW("beginRefreshMaterializedView", ""), + FINISH_REFRESH_MATERIALIZED_VIEW("finishRefreshMaterializedView", ""), + GET_REFERENCED_MATERIALIZED_VIEWS("Metadata.getReferencedMaterializedViews", ""), + RESOLVE_INDEX("resolveIndex", ""), + CREATE_ROLE("createRole", ""), + DROP_ROLE("dropRole", ""), + LIST_ROLES("listRoles", ""), + LIST_ROLE_GRANTS("listRoleGrants", ""), + GRANT_ROLES("grantRoles", ""), + REVOKE_ROLES("revokeRoles", ""), + LIST_APPLICABLE_ROLES("listApplicableRoles", ""), + LIST_ENABLED_ROLES("listEnabledRoles", ""), + GRANT_TABLE_PRIVILEGES("Metadata.grantTablePrivileges", ""), + REVOKE_TABLE_PRIVILEGES("Metadata.revokeTablePrivileges", ""), + LIST_TABLE_PRIVILEGES("Metadata.listTablePrivileges", ""), + COMMIT_PAGE_SINK_ASYNC("commitPageSinkAsync", ""), + GET_METADATA_UPDATE_RESULTS("getMetadataUpdateResults", ""), + GET_METADATA_RESOLVER("getMetadataResolver", ""), + CATALOG_EXISTS("Metadata.catalogExists", ""), + SCHEMA_EXISTS("Metadata.schemaExists", ""), + TABLE_EXISTS("Metadata.tableExists", ""), + GET_TABLE_HANDLE("Metadata.getTableHandle", ""), + GET_COLUMNS("getColumns", ""), + GET_VIEW("Metadata.getView", ""), + GET_MATERIALIZED_VIEW("Metadata.getMaterializedView", ""), + GET_CONNECTOR_CAPABILITIES("getConnectorCapabilities", ""), + GET_TABLE_LAYOUT_FILTER_COVERAGE("getTableLayoutFilterCoverage", ""), + DROP_CONSTRAINT("dropConstraint", ""), + ADD_CONSTRAINT("addConstraint", ""), + DESERIALIZE_VIEW("deserializeView", ""), + GET_CATALOG_METADATA("getCatalogMetadata", ""), + GET_CATALOG_METADATA_FOR_WRITE("Metadata.getCatalogMetadataForWrite", ""), + GET_METADATA("getMetadata", ""), + GET_METADATA_FOR_WRITE("getMetadataForWrite", ""), + CAN_RESOLVE_OPERATOR("canResolveOperator", ""), + RESOLVE_TABLE_LAYOUT("resolveTableLayout", ""), + GET_CATALOGS_BY_QUERY_ID("getCatalogsByQueryId", ""), + ADD_SYSTEM_ACCESS_CONTROL_FACTORY("addSystemAccessControlFactory", ""), + ADD_CATALOG_ACCESS_CONTROL("addCatalogAccessControl", ""), + REMOVE_CATALOG_ACCESS_CONTROL("removeCatalogAccessControl", ""), + LOAD_CATALOG_ACCESS_CONTROL("loadSystemAccessControl", ""), + STAGE("stage", ""), + LOAD_SYSTEM_ACCESS_CONTROL("loadSystemAccessControl", ""), + SET_SYSTEM_ACCESS_CONTROL("setSystemAccessControl", ""), + CHECK_CAN_SET_USER("checkCanSetUser", ""), + SELECT_AUTHORIZED_IDENTITY("selectAuthorizedIdentity", ""), + CHECK_QUERY_INTEGRITY("checkQueryIntegrity", ""), + FILTER_CATALOGS("filterCatalogs", ""), + CHECK_CAN_ACCESS_CATALOG("checkCanAccessCatalog", ""), + CHECK_CAN_CREATE_SCHEMA("checkCanCreateSchema", ""), + CHECK_CAN_DROP_SCHEMA("checkCanDropSchema", ""), + CHECK_CAN_RENAME_SCHEMA("checkCanRenameSchema", ""), + CHECK_CAN_SHOW_SCHEMAS("checkCanShowSchemas", ""), + FILTER_SCHEMAS("filterSchemas", ""), + CHECK_CAN_SHOW_CREATE_TABLE("checkCanShowCreateTable", ""), + CHECK_CAN_CREATE_TABLE("checkCanCreateTable", ""), + CHECK_CAN_DROP_TABLE("checkCanDropTable", ""), + CHECK_CAN_RENAME_TABLE("checkCanRenameTable", ""), + CHECK_CAN_SHOW_TABLES_METADATA("checkCanShowTablesMetadata", ""), + FILTER_TABLES("filterTables", ""), + CHECK_CAN_SHOW_COLUMNS_METADATA("checkCanShowColumnsMetadata", ""), + FILTER_COLUMNS("filterColumns", ""), + CHECK_CAN_ALTER_COLUMN("checkCanAlterColumn", ""), + CHECK_CAN_DROP_COLUMN("checkCanDropColumn", ""), + CHECK_CAN_INSERT_INTO_TABLE("checkCanInsertIntoTable", ""), + CHECK_CAN_RENAME_COLUMN("checkCanRenameColumn", ""), + CHECK_CAN_DELETE_FROM_TABLE("checkCanDeleteFromTable", ""), + CHECK_CAN_TRUNCATE_TABLE("checkCanTruncateTable", ""), + CHECK_CAN_UPDATE_TABLE_COLUMNS("checkCanUpdateTableColumns", ""), + CHECK_CAN_CREATE_VIEW("checkCanCreateView", ""), + CHECK_CAN_DROP_VIEW("checkCanDropView", ""), + CHECK_CAN_RENAME_VIEW("checkCanRenameView", ""), + CHECK_CAN_CREATE_VIEW_WITH_SELECT_FROM_COLUMNS("checkCanCreateViewWithSelectFromColumns", ""), + CHECK_CAN_GRANT_TABLE_PRIVILEGE("checkCanGrantTablePrivilege", ""), + CHECK_CAN_REVOKE_TABLE_PRIVILEGE("checkCanRevokeTablePrivilege", ""), + CHECK_CAN_SET_SYSTEM_SESSION_PROPERTY("checkCanSetSystemSessionProperty", ""), + CHECK_CAN_SET_CATALOG_SESSION_PROPERTY("checkCanSetCatalogSessionProperty", ""), + CHECK_CAN_SELECT_FROM_COLUMNS("checkCanSelectFromColumns", ""), + CHECK_CAN_CREATE_ROLE("checkCanCreateRole", ""), + CHECK_CAN_DROP_ROLE("checkCanDropRole", ""), + CHECK_CAN_GRANT_ROLES("checkCanGrantRoles", ""), + CHECK_CAN_REVOKE_ROLES("checkCanRevokeRoles", ""), + CHECK_CAN_SET_ROLE("checkCanSetRole", ""), + CHECK_CAN_SHOW_ROLES("checkCanShowRoles", ""), + CHECK_CAN_SHOW_CURRENT_ROLES("checkCanShowCurrentRoles", ""), + CHECK_CAN_SHOW_ROLE_GRANTS("checkCanShowRoleGrants", ""), + GET_ROW_FILTERS("getRowFilters", ""), + GET_COLUMN_MASKS("getColumnMasks", ""), + CHECK_CAN_DROP_CONSTRAINT("checkCanDropConstraint", ""), + CHECK_CAN_ADD_CONSTRAINTS("checkCanAddConstraints", ""), + GET_CONNECTOR_ACCESS_CONTROL("getConnectorAccessControl", ""); + + private final String name; + private final String description; + + TracingEnum(String name, String description) + { + this.name = name; + this.description = description; + } + + public String getName() + { + return this.name; + } + + public String getDescription() + { + return this.description; + } +} diff --git a/presto-common/src/test/java/com/facebook/presto/common/TestTelemetryConfig.java b/presto-common/src/test/java/com/facebook/presto/common/TestTelemetryConfig.java new file mode 100644 index 0000000000000..ac86f360a7fd7 --- /dev/null +++ b/presto-common/src/test/java/com/facebook/presto/common/TestTelemetryConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.common; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +public class TestTelemetryConfig +{ + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("tracing-factory.name", "otltest") + .put("tracing-enabled", "false") + .put("tracing-backend-url", "http://0.0.0.0:123") + .put("max-exporter-batch-size", "123") + .put("max-queue-size", "1234") + .put("schedule-delay", "4567") + .put("exporter-timeout", "6789") + .put("trace-sampling-ratio", "2.0") + .put("span-sampling", "false") + .build(); + + TelemetryConfig.getTelemetryConfig().setTelemetryProperties(properties); + TelemetryConfig telemetryConfig = TelemetryConfig.getTelemetryConfig(); + assertEquals(TelemetryConfig.getTracingEnabled(), false); + assertEquals(telemetryConfig.getTracingBackendUrl(), "http://0.0.0.0:123"); + assertEquals(telemetryConfig.getMaxExporterBatchSize(), 123); + assertEquals(telemetryConfig.getMaxQueueSize(), 1234); + assertEquals(telemetryConfig.getScheduleDelay(), 4567); + assertEquals(telemetryConfig.getExporterTimeout(), 6789); + assertEquals(telemetryConfig.getSamplingRatio(), 2.0); + assertEquals(TelemetryConfig.getSpanSampling(), false); + } +} diff --git a/presto-docs/src/main/sphinx/plugin.rst b/presto-docs/src/main/sphinx/plugin.rst index 7f4e7883028e4..888fb304a6e7a 100644 --- a/presto-docs/src/main/sphinx/plugin.rst +++ b/presto-docs/src/main/sphinx/plugin.rst @@ -8,4 +8,5 @@ This chapter outlines the plugins in Presto that are available for various use c :maxdepth: 1 plugin/redis-hbo-provider + plugin/open-telemetry diff --git a/presto-docs/src/main/sphinx/plugin/open-telemetry.rst b/presto-docs/src/main/sphinx/plugin/open-telemetry.rst new file mode 100644 index 0000000000000..dc86b369cce52 --- /dev/null +++ b/presto-docs/src/main/sphinx/plugin/open-telemetry.rst @@ -0,0 +1,27 @@ +============== +Open Telemetry +============== +OpenTelemetry is a powerful serviceability framework that helps us gain insights into the performance and behavior of the systems. It facilitates generation, collection, and management of telemetry data +such as traces and metrics to observability dashboards. + +Configuration +------------- + +To enable this feature set ``tracing-enabled`` to ``true`` and set the path for ``tracing-backend-url`` in ``etc/telemetry-tracing.properties`` . + +Configuration properties +------------------------ + +============================================ ================================================================================================ ================== +Property Name Description Default Values +============================================ ================================================================================================ ================== +``tracing-factory.name`` Unique identifier for factory implementation to be registered. otel +``tracing-enabled`` Boolean value controlling if tracing is on or off. false +``tracing-backend-url`` URL of the backend for exporting telemetry data. +``max-exporter-batch-size`` Maximum number of spans to export in one batch. 256 +``max-queue-size`` Maximum number of spans to queue before processing for export. 1024 +``schedule-delay`` Delay between batches of span export, controlling how frequently spans are exported. 1000 +``exporter-timeout`` How long the span exporter waits for a batch of spans to be successfully sent before timing out. 1024 +``trace-sampling-ratio`` Double between 0.0 and 1.0 to specify the percentage of queries to be traced. 1.0 +``span-sampling`` Boolean to enable/disable sampling. If enabled, spans are only generated for major operations. false +============================================ ================================================================================================ ================== \ No newline at end of file diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 28c4fc5146388..2d58226629b02 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -1165,12 +1165,6 @@ public Locale getLocale() return session.getLocale(); } - @Override - public Optional getTraceToken() - { - return session.getTraceToken(); - } - @Override public Optional getClientInfo() { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java index 8cf90df570ccb..afa6f0c0c3e24 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/HiveQueryRunner.java @@ -193,8 +193,6 @@ public static DistributedQueryRunner createQueryRunner( Map systemProperties = ImmutableMap.builder() .put("task.writer-count", "2") .put("task.partitioned-writer-count", "4") - .put("tracing.tracer-type", "simple") - .put("tracing.enable-distributed-tracing", "simple") .putAll(extraProperties) .build(); diff --git a/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java b/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java index 1f3f760631850..14dc8f341bbaf 100644 --- a/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java +++ b/presto-jdbc/src/main/java/com/facebook/presto/jdbc/PrestoConnection.java @@ -764,7 +764,6 @@ else if (applicationName != null) { source = applicationName; } - Optional traceToken = Optional.ofNullable(clientInfo.get("TraceToken")); Iterable clientTags = Splitter.on(',').trimResults().omitEmptyStrings() .split(nullToEmpty(clientInfo.get("ClientTags"))); @@ -779,7 +778,6 @@ else if (applicationName != null) { httpUri, user, source, - traceToken, ImmutableSet.copyOf(clientTags), clientInfo.get("ClientInfo"), catalog.get(), diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java index 17e64f99121ab..a3c27a645c3fc 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDatabaseMetaData.java @@ -165,7 +165,6 @@ public void testGetClientInfoProperties() assertEquals(resultSet.getString(4), "Comma-delimited string of tags for the session"); assertTrue(resultSet.next()); - assertEquals(resultSet.getString(1), "TraceToken"); assertEquals(resultSet.getInt(2), MAX_LENGTH); assertNull(resultSet.getString(3)); assertEquals(resultSet.getString(4), "Sets the trace token of the session"); diff --git a/presto-main/etc/config.properties b/presto-main/etc/config.properties index a120422c4b6f1..72f1ee017a711 100644 --- a/presto-main/etc/config.properties +++ b/presto-main/etc/config.properties @@ -51,7 +51,8 @@ plugin.bundles=\ ../presto-node-ttl-fetchers/pom.xml,\ ../presto-hive-function-namespace/pom.xml,\ ../presto-delta/pom.xml,\ - ../presto-hudi/pom.xml + ../presto-hudi/pom.xml,\ + ../presto-open-telemetry/pom.xml presto.version=testversion node-scheduler.include-coordinator=true diff --git a/presto-main/etc/log.properties b/presto-main/etc/log.properties index d9790f7ed5795..6c821bf33ab4b 100644 --- a/presto-main/etc/log.properties +++ b/presto-main/etc/log.properties @@ -9,3 +9,4 @@ com.facebook.presto=INFO com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory=WARN com.ning.http.client=WARN com.facebook.presto.server.PluginManager=DEBUG +io.opentelemetry=DEBUG \ No newline at end of file diff --git a/presto-main/etc/telemetry-tracing.properties b/presto-main/etc/telemetry-tracing.properties new file mode 100644 index 0000000000000..6f7b58c8fe01e --- /dev/null +++ b/presto-main/etc/telemetry-tracing.properties @@ -0,0 +1,18 @@ +# unique identifier for factory implementation to be registered +tracing-factory.name=otel +# boolean value controlling if tracing is on or off +tracing-enabled=false +# points to backend for exporting telemetry data +tracing-backend-url=http://localhost:4317 +# maximum number of spans that will be exported in one batch +max-exporter-batch-size=256 +# maximum number of spans that can be queued before being processed for export +max-queue-size=1024 +# delay between batches of span export, controlling how frequently spans are exported +schedule-delay=1000 +# how long the span exporter will wait for a batch of spans to be successfully sent before timing out +exporter-timeout=1024 +# double between 0.0 and 1.0 to specify the percentage of queries to be traced +trace-sampling-ratio=1.0 +# boolean to enable/disable sampling. If enabled, spans are only generated for major operations +span-sampling=false \ No newline at end of file diff --git a/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java b/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java index 2ece57e7c1811..fd7898000dc2f 100644 --- a/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java +++ b/presto-main/src/main/java/com/facebook/presto/FullConnectorSession.java @@ -120,12 +120,6 @@ public long getStartTime() return session.getStartTime(); } - @Override - public Optional getTraceToken() - { - return session.getTraceToken(); - } - @Override public Optional getClientInfo() { @@ -179,7 +173,6 @@ public String toString() .add("queryId", getQueryId()) .add("user", getUser()) .add("source", getSource().orElse(null)) - .add("traceToken", getTraceToken().orElse(null)) .add("timeZoneKey", getTimeZoneKey()) .add("locale", getLocale()) .add("startTime", getStartTime()) diff --git a/presto-main/src/main/java/com/facebook/presto/Session.java b/presto-main/src/main/java/com/facebook/presto/Session.java index 8ccca2db4d3de..6349387ba0038 100644 --- a/presto-main/src/main/java/com/facebook/presto/Session.java +++ b/presto-main/src/main/java/com/facebook/presto/Session.java @@ -35,10 +35,11 @@ import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.spi.session.ResourceEstimates; import com.facebook.presto.spi.session.SessionPropertyConfigurationManager.SystemSessionPropertyConfiguration; -import com.facebook.presto.spi.tracing.Tracer; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.analyzer.CTEInformationCollector; import com.facebook.presto.sql.planner.optimizations.OptimizerInformationCollector; import com.facebook.presto.sql.planner.optimizations.OptimizerResultCollector; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -75,6 +76,8 @@ public final class Session { private final QueryId queryId; + private final BaseSpan querySpan; + private final BaseSpan rootSpan; private final Optional transactionId; private final boolean clientTransactionSupport; private final Identity identity; @@ -86,7 +89,6 @@ public final class Session private final Optional remoteUserAddress; private final Optional userAgent; private final Optional clientInfo; - private final Optional traceToken; private final Set clientTags; private final ResourceEstimates resourceEstimates; private final long startTime; @@ -97,7 +99,6 @@ public final class Session private final Map preparedStatements; private final Map sessionFunctions; private final AccessControlContext context; - private final Optional tracer; private final WarningCollector warningCollector; private final RuntimeStats runtimeStats; private final Optional queryType; @@ -110,13 +111,14 @@ public final class Session public Session( QueryId queryId, + BaseSpan querySpan, + BaseSpan rootSpan, Optional transactionId, boolean clientTransactionSupport, Identity identity, Optional source, Optional catalog, Optional schema, - Optional traceToken, TimeZoneKey timeZoneKey, Locale locale, Optional remoteUserAddress, @@ -131,19 +133,19 @@ public Session( SessionPropertyManager sessionPropertyManager, Map preparedStatements, Map sessionFunctions, - Optional tracer, WarningCollector warningCollector, RuntimeStats runtimeStats, Optional queryType) { this.queryId = requireNonNull(queryId, "queryId is null"); + this.querySpan = querySpan; + this.rootSpan = rootSpan; this.transactionId = requireNonNull(transactionId, "transactionId is null"); this.clientTransactionSupport = clientTransactionSupport; this.identity = requireNonNull(identity, "identity is null"); this.source = requireNonNull(source, "source is null"); this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); - this.traceToken = requireNonNull(traceToken, "traceToken is null"); this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); this.locale = requireNonNull(locale, "locale is null"); this.remoteUserAddress = requireNonNull(remoteUserAddress, "remoteUserAddress is null"); @@ -172,7 +174,6 @@ public Session( checkArgument(!transactionId.isPresent() || unprocessedCatalogProperties.isEmpty(), "Catalog session properties cannot be set if there is an open transaction"); checkArgument(catalog.isPresent() || !schema.isPresent(), "schema is set but catalog is not"); - this.tracer = requireNonNull(tracer, "tracer is null"); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); this.runtimeStats = requireNonNull(runtimeStats, "runtimeStats is null"); this.queryType = requireNonNull(queryType, "queryType is null"); @@ -184,6 +185,16 @@ public QueryId getQueryId() return queryId; } + public BaseSpan getQuerySpan() + { + return querySpan; + } + + public BaseSpan getRootSpan() + { + return rootSpan; + } + public String getUser() { return identity.getUser(); @@ -239,11 +250,6 @@ public Set getClientTags() return clientTags; } - public Optional getTraceToken() - { - return traceToken; - } - public ResourceEstimates getResourceEstimates() { return resourceEstimates; @@ -322,11 +328,6 @@ public RuntimeStats getRuntimeStats() return runtimeStats; } - public Optional getTracer() - { - return tracer; - } - public WarningCollector getWarningCollector() { return warningCollector; @@ -426,6 +427,8 @@ public Session beginTransactionId(TransactionId transactionId, TransactionManage return new Session( queryId, + querySpan, + rootSpan, Optional.of(transactionId), clientTransactionSupport, new Identity( @@ -439,7 +442,6 @@ public Session beginTransactionId(TransactionId transactionId, TransactionManage source, catalog, schema, - traceToken, timeZoneKey, locale, remoteUserAddress, @@ -454,7 +456,6 @@ public Session beginTransactionId(TransactionId transactionId, TransactionManage sessionPropertyManager, preparedStatements, sessionFunctions, - tracer, warningCollector, runtimeStats, queryType); @@ -508,7 +509,6 @@ public SessionRepresentation toSessionRepresentation() source, catalog, schema, - traceToken, timeZoneKey, locale, remoteUserAddress, @@ -530,13 +530,14 @@ public String toString() { return toStringHelper(this) .add("queryId", queryId) + .add("querySpan", TracingManager.spanString(querySpan).orElse(null)) + .add("rootSpan", rootSpan.toString()) .add("transactionId", transactionId) .add("user", getUser()) .add("principal", getIdentity().getPrincipal().orElse(null)) .add("source", source.orElse(null)) .add("catalog", catalog.orElse(null)) .add("schema", schema.orElse(null)) - .add("traceToken", traceToken.orElse(null)) .add("timeZoneKey", timeZoneKey) .add("locale", locale) .add("remoteUserAddress", remoteUserAddress.orElse(null)) @@ -562,13 +563,14 @@ public static SessionBuilder builder(Session session) public static class SessionBuilder { private QueryId queryId; + private BaseSpan querySpan = TracingManager.getInvalidSpan(); //do not initialize with null + private BaseSpan rootSpan = TracingManager.getInvalidSpan(); //do not initialize with null private TransactionId transactionId; private boolean clientTransactionSupport; private Identity identity; private String source; private String catalog; private String schema; - private Optional traceToken = Optional.empty(); private TimeZoneKey timeZoneKey = TimeZoneKey.getTimeZoneKey(TimeZone.getDefault().getID()); private Locale locale = Locale.getDefault(); private String remoteUserAddress; @@ -576,7 +578,6 @@ public static class SessionBuilder private String clientInfo; private Set clientTags = ImmutableSet.of(); private ResourceEstimates resourceEstimates; - private Optional tracer = Optional.empty(); private long startTime = System.currentTimeMillis(); private final Map systemProperties = new HashMap<>(); private final Map> connectorProperties = new HashMap<>(); @@ -605,7 +606,6 @@ private SessionBuilder(Session session) this.source = session.source.orElse(null); this.catalog = session.catalog.orElse(null); this.schema = session.schema.orElse(null); - this.traceToken = requireNonNull(session.traceToken, "traceToken is null"); this.timeZoneKey = session.timeZoneKey; this.locale = session.locale; this.remoteUserAddress = session.remoteUserAddress.orElse(null); @@ -617,7 +617,6 @@ private SessionBuilder(Session session) session.unprocessedCatalogProperties.forEach((key, value) -> this.catalogSessionProperties.put(key, new HashMap<>(value))); this.preparedStatements.putAll(session.preparedStatements); this.sessionFunctions.putAll(session.sessionFunctions); - this.tracer = requireNonNull(session.tracer, "tracer is null"); this.warningCollector = requireNonNull(session.warningCollector, "warningCollector is null"); this.runtimeStats = requireNonNull(session.runtimeStats, "runtimeStats is null"); this.queryType = requireNonNull(session.queryType, "queryType is null"); @@ -636,12 +635,6 @@ public SessionBuilder setTransactionId(TransactionId transactionId) return this; } - public SessionBuilder setTracer(Optional tracer) - { - this.tracer = requireNonNull(tracer, "tracer is null"); - return this; - } - public SessionBuilder setClientTransactionSupport() { this.clientTransactionSupport = true; @@ -678,12 +671,6 @@ public SessionBuilder setSource(String source) return this; } - public SessionBuilder setTraceToken(Optional traceToken) - { - this.traceToken = requireNonNull(traceToken, "traceToken is null"); - return this; - } - public SessionBuilder setStartTime(long startTime) { this.startTime = startTime; @@ -742,6 +729,18 @@ public SessionBuilder setConnectionProperty(ConnectorId connectorId, String prop return this; } + public SessionBuilder setQuerySpan(BaseSpan querySpan) + { + this.querySpan = querySpan; + return this; + } + + public SessionBuilder setRootSpan(BaseSpan rootSpan) + { + this.rootSpan = rootSpan; + return this; + } + /** * Sets a catalog property for the session. The property name and value must * only contain characters from US-ASCII and must not be for '='. @@ -837,13 +836,14 @@ public Session build() { return new Session( queryId, + querySpan, + rootSpan, Optional.ofNullable(transactionId), clientTransactionSupport, identity, Optional.ofNullable(source), Optional.ofNullable(catalog), Optional.ofNullable(schema), - traceToken, timeZoneKey, locale, Optional.ofNullable(remoteUserAddress), @@ -859,7 +859,6 @@ public Session build() sessionPropertyManager, preparedStatements, sessionFunctions, - tracer, warningCollector, runtimeStats, queryType); diff --git a/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java b/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java index 7651ab2b96489..bc1e33b634ecf 100644 --- a/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java +++ b/presto-main/src/main/java/com/facebook/presto/SessionRepresentation.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.spi.security.TokenAuthenticator; import com.facebook.presto.spi.session.ResourceEstimates; +import com.facebook.presto.telemetry.TracingManager; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; @@ -54,7 +55,6 @@ public final class SessionRepresentation private final Optional source; private final Optional catalog; private final Optional schema; - private final Optional traceToken; private final TimeZoneKey timeZoneKey; private final Locale locale; private final Optional remoteUserAddress; @@ -81,7 +81,6 @@ public SessionRepresentation( @JsonProperty("source") Optional source, @JsonProperty("catalog") Optional catalog, @JsonProperty("schema") Optional schema, - @JsonProperty("traceToken") Optional traceToken, @JsonProperty("timeZoneKey") TimeZoneKey timeZoneKey, @JsonProperty("locale") Locale locale, @JsonProperty("remoteUserAddress") Optional remoteUserAddress, @@ -105,7 +104,6 @@ public SessionRepresentation( this.source = requireNonNull(source, "source is null"); this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); - this.traceToken = requireNonNull(traceToken, "traceToken is null"); this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); this.locale = requireNonNull(locale, "locale is null"); this.remoteUserAddress = requireNonNull(remoteUserAddress, "remoteUserAddress is null"); @@ -188,13 +186,6 @@ public Optional getSchema() return schema; } - @ThriftField(9) - @JsonProperty - public Optional getTraceToken() - { - return traceToken; - } - @ThriftField(10) @JsonProperty public TimeZoneKey getTimeZoneKey() @@ -307,6 +298,8 @@ public Session toSession(SessionPropertyManager sessionPropertyManager, MapResource Groups */ - public ListenableFuture createQuery(QueryId queryId, String slug, int retryCount, SessionContext sessionContext, String query) + public ListenableFuture createQuery(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, String slug, int retryCount, SessionContext sessionContext, String query) { requireNonNull(queryId, "queryId is null"); requireNonNull(sessionContext, "sessionFactory is null"); @@ -242,14 +246,15 @@ public ListenableFuture createQuery(QueryId queryId, String slug, int retryCo checkArgument(!queryTracker.tryGetQuery(queryId).isPresent(), "query %s already exists", queryId); DispatchQueryCreationFuture queryCreationFuture = new DispatchQueryCreationFuture(); - boundedQueryExecutor.execute(() -> { - try { - createQueryInternal(queryId, slug, retryCount, sessionContext, query, resourceGroupManager); + + boundedQueryExecutor.execute(TracingManager.getCurrentContextWrap(() -> { + try (BaseSpan ignored = scopedSpan(querySpan, TracingEnum.DISPATCH.getName())) { + createQueryInternal(queryId, querySpan, rootSpan, slug, retryCount, sessionContext, query, resourceGroupManager); } finally { queryCreationFuture.set(null); } - }); + })); return queryCreationFuture; } @@ -257,7 +262,7 @@ public ListenableFuture createQuery(QueryId queryId, String slug, int retryCo * Creates and registers a dispatch query with the query tracker. This method will never fail to register a query with the query * tracker. If an error occurs while creating a dispatch query, a failed dispatch will be created and registered. */ - private void createQueryInternal(QueryId queryId, String slug, int retryCount, SessionContext sessionContext, String query, ResourceGroupManager resourceGroupManager) + private void createQueryInternal(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, String slug, int retryCount, SessionContext sessionContext, String query, ResourceGroupManager resourceGroupManager) { Session session = null; SessionBuilder sessionBuilder = null; @@ -270,7 +275,7 @@ private void createQueryInternal(QueryId queryId, String slug, int retryCoun } // decode session - sessionBuilder = sessionSupplier.createSessionBuilder(queryId, sessionContext, warningCollectorFactory); + sessionBuilder = sessionSupplier.createSessionBuilder(queryId, querySpan, rootSpan, sessionContext, warningCollectorFactory); session = sessionBuilder.build(); // prepare query @@ -338,6 +343,8 @@ private void createQueryInternal(QueryId queryId, String slug, int retryCoun } DispatchQuery failedDispatchQuery = failedDispatchQueryFactory.createFailedDispatchQuery(session, query, Optional.empty(), throwable); queryCreated(failedDispatchQuery); + + TracingManager.endSpanOnError(querySpan, throwable); } } diff --git a/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java b/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java index 14981c4f33fed..e6c036c345a9a 100644 --- a/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java @@ -31,8 +31,6 @@ import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.AnalyzerProviderManager; -import com.facebook.presto.tracing.NoopTracerProvider; -import com.facebook.presto.tracing.QueryStateTracingListener; import com.facebook.presto.transaction.TransactionManager; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -158,7 +156,6 @@ public DispatchQuery createDispatchQuery( metadata, warningCollector); - stateMachine.addStateChangeListener(new QueryStateTracingListener(stateMachine.getSession().getTracer().orElse(NoopTracerProvider.NOOP_TRACER))); queryMonitor.queryCreatedEvent(stateMachine.getBasicQueryInfo(Optional.empty())); ListenableFuture queryExecutionFuture = executor.submit(() -> { diff --git a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java index 1998c567ebb04..d16dd796a076c 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/QueryMonitor.java @@ -166,8 +166,7 @@ public void queryCreatedEvent(BasicQueryInfo queryInfo) Optional.empty(), Optional.empty(), Optional.empty(), - ImmutableList.of(), - queryInfo.getSession().getTraceToken()))); + ImmutableList.of()))); } public void queryUpdatedEvent(QueryInfo queryInfo) @@ -191,8 +190,7 @@ public void publishQueryProgressEvent(long monotonicallyIncreasingEventId, Basic Optional.empty(), Optional.empty(), Optional.empty(), - ImmutableList.of(), - queryInfo.getSession().getTraceToken()), + ImmutableList.of()), createQueryStatistics(queryInfo), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), queryInfo.getQueryType(), @@ -214,8 +212,7 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur Optional.empty(), Optional.empty(), Optional.empty(), - ImmutableList.of(), - queryInfo.getSession().getTraceToken()), + ImmutableList.of()), new QueryStatistics( ofMillis(0), ofMillis(0), @@ -356,8 +353,7 @@ private QueryMetadata createQueryMetadata(QueryInfo queryInfo) queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, maxJsonLimit)), queryInfo.getRuntimeOptimizedStages().orElse(ImmutableList.of()).stream() .map(stageId -> String.valueOf(stageId.getId())) - .collect(toImmutableList()), - queryInfo.getSession().getTraceToken()); + .collect(toImmutableList())); } private List createOperatorStatistics(QueryInfo queryInfo) diff --git a/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java b/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java index 6a0296aac5499..1e875e30f171b 100644 --- a/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java +++ b/presto-main/src/main/java/com/facebook/presto/event/SplitMonitor.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; import com.facebook.presto.spi.eventlistener.SplitFailureInfo; import com.facebook.presto.spi.eventlistener.SplitStatistics; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,6 +28,7 @@ import javax.inject.Inject; import java.time.Duration; +import java.util.Objects; import java.util.Optional; import static java.time.Duration.ofMillis; @@ -46,17 +48,17 @@ public SplitMonitor(EventListenerManager eventListenerManager, ObjectMapper obje this.objectMapper = requireNonNull(objectMapper, "objectMapper is null"); } - public void splitCompletedEvent(TaskId taskId, DriverStats driverStats) + public void splitCompletedEvent(TaskId taskId, DriverStats driverStats, BaseSpan pipelineSpan) { - splitCompletedEvent(taskId, driverStats, null, null); + splitCompletedEvent(taskId, driverStats, null, null, pipelineSpan); } - public void splitFailedEvent(TaskId taskId, DriverStats driverStats, Throwable cause) + public void splitFailedEvent(TaskId taskId, DriverStats driverStats, Throwable cause, BaseSpan pipelineSpan) { - splitCompletedEvent(taskId, driverStats, cause.getClass().getName(), cause.getMessage()); + splitCompletedEvent(taskId, driverStats, cause.getClass().getName(), cause.getMessage(), pipelineSpan); } - private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage) + private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullable String failureType, @Nullable String failureMessage, BaseSpan pipelineSpan) { Optional timeToStart = Optional.empty(); if (driverStats.getStartTime() != null) { @@ -93,10 +95,14 @@ private void splitCompletedEvent(TaskId taskId, DriverStats driverStats, @Nullab timeToStart, timeToEnd), splitFailureMetadata, - objectMapper.writeValueAsString(driverStats))); + objectMapper.writeValueAsString(driverStats)), + pipelineSpan); } catch (JsonProcessingException e) { log.error(e, "Error processing split completion event for task %s", taskId); } + if (!Objects.isNull(pipelineSpan)) { + pipelineSpan.end(); + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java b/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java index 24a1d04b54bb1..2d1d4b4a3d73f 100644 --- a/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java +++ b/presto-main/src/main/java/com/facebook/presto/eventlistener/EventListenerManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.eventlistener; import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.spi.eventlistener.EventListenerFactory; @@ -22,6 +23,7 @@ import com.facebook.presto.spi.eventlistener.QueryProgressEvent; import com.facebook.presto.spi.eventlistener.QueryUpdatedEvent; import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -32,6 +34,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; import static com.facebook.presto.util.PropertiesUtil.loadProperties; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -122,9 +125,11 @@ public void publishQueryProgress(QueryProgressEvent queryProgressEvent) .ifPresent(eventListener -> eventListener.publishQueryProgress(queryProgressEvent)); } - public void splitCompleted(SplitCompletedEvent splitCompletedEvent) + public void splitCompleted(SplitCompletedEvent splitCompletedEvent, BaseSpan pipelineSpan) { - configuredEventListener.get() - .ifPresent(eventListener -> eventListener.splitCompleted(splitCompletedEvent)); + try (BaseSpan ignored = scopedSpan(pipelineSpan, TracingEnum.SPLIT.getName(), ImmutableMap.of("QUERY_ID", splitCompletedEvent.getQueryId(), "STAGE_ID", splitCompletedEvent.getStageId(), "TASK_ID", splitCompletedEvent.getTaskId(), "START_TIME", splitCompletedEvent.getStartTime().map(String::valueOf).orElse(""), "END_TIME", splitCompletedEvent.getEndTime().map(String::valueOf).orElse(""), "PAYLOAD", splitCompletedEvent.getPayload(), "FAILURE_INFO", splitCompletedEvent.getFailureInfo().map(String::valueOf).orElse("")))) { + configuredEventListener.get() + .ifPresent(eventListener -> eventListener.splitCompleted(splitCompletedEvent)); + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java index b22822fc0149e..ee26819736ac3 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java @@ -42,8 +42,10 @@ import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.spi.statistics.ColumnStatistics; import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.CanonicalPlanWithInfo; import com.facebook.presto.sql.planner.PlanFragment; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.transaction.TransactionInfo; import com.facebook.presto.transaction.TransactionManager; import com.google.common.base.Ticker; @@ -67,6 +69,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -95,6 +98,7 @@ import static com.facebook.presto.spi.StandardErrorCode.INVALID_ARGUMENTS; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.USER_CANCELED; +import static com.facebook.presto.telemetry.TracingManager.addEvent; import static com.facebook.presto.util.Failures.toFailure; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -262,6 +266,11 @@ static QueryStateMachine beginWithTicker( session = session.beginTransactionId(transactionId, transactionManager, accessControl); } + BaseSpan querySpan = session.getQuerySpan(); + BaseSpan rootSpan = session.getRootSpan(); + + TracingManager.setAttributes(querySpan, ImmutableMap.of("QUERY_TYPE", queryType.map(Enum::name).orElse("UNKNOWN"))); + QueryStateMachine queryStateMachine = new QueryStateMachine( query, preparedQuery, @@ -278,8 +287,32 @@ static QueryStateMachine beginWithTicker( queryStateMachine.addStateChangeListener(newState -> { QUERY_STATE_LOG.debug("Query %s is %s", queryStateMachine.getQueryId(), newState); // mark finished or failed transaction as inactive + + addEvent(querySpan, "query_state", newState.toString()); if (newState.isDone()) { - queryStateMachine.getSession().getTransactionId().ifPresent(transactionManager::trySetInactive); + try { + queryStateMachine.getSession().getTransactionId().ifPresent(transactionManager::trySetInactive); + + queryStateMachine.getFailureInfo().ifPresent( + failure -> { + ErrorCode errorCode = requireNonNull(failure.getErrorCode()); + + TracingManager.recordException(querySpan, failure.getMessage(), failure.toException(), errorCode); + }); + + queryStateMachine.getFailureInfo().orElseGet(() -> { + TracingManager.setSuccess(querySpan); + return null; + }); + } + finally { + if (!Objects.isNull(querySpan)) { + querySpan.end(); + } + if (!Objects.isNull(rootSpan)) { + rootSpan.end(); + } + } } }); return queryStateMachine; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java index d6004387b745f..990be28409ae7 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/RemoteTaskFactory.java @@ -19,6 +19,7 @@ import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.Multimap; @@ -35,5 +36,6 @@ RemoteTask createRemoteTask(Session session, NodeStatsTracker nodeStatsTracker, boolean summarizeTaskInfo, TableWriteInfo tableWriteInfo, - SchedulerStatsTracker schedulerStatsTracker); + SchedulerStatsTracker schedulerStatsTracker, + BaseSpan stagespan); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SplitRunner.java b/presto-main/src/main/java/com/facebook/presto/execution/SplitRunner.java index ded9bd28c1d49..16b3405a09771 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SplitRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SplitRunner.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.Duration; @@ -29,4 +30,9 @@ public interface SplitRunner @Override void close(); + + default BaseSpan getPipelineSpan() + { + return null; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java index 9efb9780e4e2e..3ff8836248436 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java @@ -17,6 +17,7 @@ import com.facebook.presto.Session; import com.facebook.presto.common.analyzer.PreparedQuery; import com.facebook.presto.common.resourceGroups.QueryType; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.cost.CostCalculator; import com.facebook.presto.cost.HistoryBasedPlanStatisticsManager; import com.facebook.presto.cost.StatsCalculator; @@ -47,6 +48,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.resourceGroups.ResourceGroupQueryLimits; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.split.CloseableSplitSourceProvider; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.Optimizer; @@ -101,6 +103,7 @@ import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.planner.PlanNodeCanonicalInfo.getCanonicalInfo; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static com.facebook.presto.util.AnalyzerUtil.getAnalyzerContext; import static com.google.common.base.Preconditions.checkArgument; @@ -206,15 +209,20 @@ private SqlQueryExecution( stateMachine.beginSemanticAnalyzing(); - try (TimeoutThread unused = new TimeoutThread( - Thread.currentThread(), - timeoutThreadExecutor, - getQueryAnalyzerTimeout(getSession()))) { - this.queryAnalysis = getSession() - .getRuntimeStats() - .recordWallAndCpuTime(ANALYZE_TIME_NANOS, () -> queryAnalyzer.analyze(analyzerContext, preparedQuery)); + BaseSpan querySpan = getSession().getQuerySpan(); + try (BaseSpan spanIgnored = scopedSpan(querySpan, TracingEnum.ANALYZER.getName())) { + try (TimeoutThread unused = new TimeoutThread( + Thread.currentThread(), + timeoutThreadExecutor, + getQueryAnalyzerTimeout(getSession()))) { + this.queryAnalysis = getSession() + .getRuntimeStats() + .recordWallAndCpuTime(ANALYZE_TIME_NANOS, () -> queryAnalyzer.analyze(analyzerContext, preparedQuery)); + } } + stateMachine.beginSemanticAnalyzing(); + stateMachine.setUpdateType(queryAnalysis.getUpdateType()); stateMachine.setExpandedQuery(queryAnalysis.getExpandedQuery()); @@ -569,44 +577,66 @@ private PlanRoot doCreateLogicalPlanAndOptimize() LOGICAL_PLANNER_TIME_NANOS, () -> queryAnalyzer.plan(this.analyzerContext, queryAnalysis)); - Optimizer optimizer = new Optimizer( - stateMachine.getSession(), - metadata, - planOptimizers, - planChecker, - analyzerContext.getVariableAllocator(), - idAllocator, - stateMachine.getWarningCollector(), - statsCalculator, - costCalculator, - false); + BaseSpan querySpan = getSession().getQuerySpan(); + try (BaseSpan ignored = scopedSpan(querySpan, TracingEnum.PLANNER.getName())) { + return optimizePlan(planNode); + } + } + catch (StackOverflowError e) { + throw new PrestoException(NOT_SUPPORTED, "statement is too large (stack overflow during analysis)", e); + } + } + + private PlanRoot optimizePlan(PlanNode planNode) + { + Optimizer optimizer = new Optimizer( + stateMachine.getSession(), + metadata, + planOptimizers, + planChecker, + analyzerContext.getVariableAllocator(), + idAllocator, + stateMachine.getWarningCollector(), + statsCalculator, + costCalculator, + false); - Plan plan = getSession().getRuntimeStats().recordWallAndCpuTime( + Plan plan; + try (BaseSpan ignored = scopedSpan("Plan Optimizer")) { + plan = getSession().getRuntimeStats().recordWallAndCpuTime( OPTIMIZER_TIME_NANOS, () -> optimizer.validateAndOptimizePlan(planNode, OPTIMIZED_AND_VALIDATED)); + } - queryPlan.set(plan); - stateMachine.setPlanStatsAndCosts(plan.getStatsAndCosts()); - stateMachine.setPlanIdNodeMap(plan.getPlanIdNodeMap()); - List canonicalPlanWithInfos = getSession().getRuntimeStats().recordWallAndCpuTime( - GET_CANONICAL_INFO_TIME_NANOS, - () -> getCanonicalInfo(getSession(), plan.getRoot(), planCanonicalInfoProvider)); - stateMachine.setPlanCanonicalInfo(canonicalPlanWithInfos); + queryPlan.set(plan); + stateMachine.setPlanStatsAndCosts(plan.getStatsAndCosts()); + stateMachine.setPlanIdNodeMap(plan.getPlanIdNodeMap()); + List canonicalPlanWithInfos = getSession().getRuntimeStats().recordWallAndCpuTime( + GET_CANONICAL_INFO_TIME_NANOS, + () -> getCanonicalInfo(getSession(), plan.getRoot(), planCanonicalInfoProvider)); + stateMachine.setPlanCanonicalInfo(canonicalPlanWithInfos); - // extract inputs + // extract inputs + try (BaseSpan ignored = scopedSpan("extract-inputs")) { List inputs = new InputExtractor(metadata, stateMachine.getSession()).extractInputs(plan.getRoot()); stateMachine.setInputs(inputs); + } - // extract output + // extract output + try (BaseSpan ignored = scopedSpan("extract-outputs")) { Optional output = new OutputExtractor().extractOutput(plan.getRoot()); stateMachine.setOutput(output); // fragment the plan // the variableAllocator is finally passed to SqlQueryScheduler for runtime cost-based optimizations variableAllocator.set(new VariableAllocator(plan.getTypes().allVariables())); - SubPlan fragmentedPlan = getSession().getRuntimeStats().recordWallAndCpuTime( - FRAGMENT_PLAN_TIME_NANOS, - () -> planFragmenter.createSubPlans(stateMachine.getSession(), plan, false, idAllocator, variableAllocator.get(), stateMachine.getWarningCollector())); + SubPlan fragmentedPlan; + + try (BaseSpan spanIgnored = scopedSpan("fragment-plan")) { + fragmentedPlan = getSession().getRuntimeStats().recordWallAndCpuTime( + FRAGMENT_PLAN_TIME_NANOS, + () -> planFragmenter.createSubPlans(stateMachine.getSession(), plan, false, idAllocator, variableAllocator.get(), stateMachine.getWarningCollector())); + } // record analysis time stateMachine.endAnalysis(); @@ -614,9 +644,6 @@ private PlanRoot doCreateLogicalPlanAndOptimize() boolean explainAnalyze = queryAnalysis.isExplainAnalyzeQuery(); return new PlanRoot(fragmentedPlan, !explainAnalyze, queryAnalysis.extractConnectors()); } - catch (StackOverflowError e) { - throw new PrestoException(NOT_SUPPORTED, "statement is too large (stack overflow during analysis)", e); - } } private PlanRoot runCreateLogicalPlanAsync() diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java index 236ab128ef59a..712a57938d8bb 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.execution; +import com.facebook.airlift.concurrent.SetThreadName; import com.facebook.airlift.concurrent.ThreadPoolExecutorMBean; import com.facebook.airlift.log.Logger; import com.facebook.presto.ExceededCpuLimitException; @@ -20,6 +21,7 @@ import com.facebook.presto.ExceededOutputSizeLimitException; import com.facebook.presto.ExceededScanLimitException; import com.facebook.presto.Session; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.cost.HistoryBasedPlanStatisticsManager; import com.facebook.presto.cost.HistoryBasedPlanStatisticsTracker; import com.facebook.presto.event.QueryMonitor; @@ -32,6 +34,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.resourceGroups.ResourceGroupQueryLimits; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.version.EmbedVersion; import com.google.common.annotations.VisibleForTesting; @@ -73,6 +76,7 @@ import static com.facebook.presto.execution.QueryState.RUNNING; import static com.facebook.presto.spi.StandardErrorCode.EXCEEDED_OUTPUT_POSITIONS_LIMIT; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static java.lang.String.format; @@ -316,7 +320,12 @@ public void createQuery(QueryExecution queryExecution) // TODO(pranjalssh): Support plan statistics tracking for other query managers historyBasedPlanStatisticsTracker.updateStatistics(queryExecution); - embedVersion.embedVersion(queryExecution::start).run(); + BaseSpan querySpan = queryExecution.getSession().getQuerySpan(); + try (SetThreadName ignored = new SetThreadName("Query-%s", queryExecution.getQueryId())) { + try (BaseSpan ignoredStartScope = scopedSpan(querySpan, TracingEnum.QUERY_START.getName())) { + embedVersion.embedVersion(queryExecution::start).run(); + } + } } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java index 2c6b7e8b60dea..60ba5ebdc87ae 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlStageExecution.java @@ -32,6 +32,7 @@ import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.split.RemoteSplit; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; @@ -158,7 +159,8 @@ public static SqlStageExecution createSqlStageExecution( ExecutorService executor, FailureDetector failureDetector, SplitSchedulerStats schedulerStats, - TableWriteInfo tableWriteInfo) + TableWriteInfo tableWriteInfo, + BaseSpan schedulerSpan) { requireNonNull(stageExecutionId, "stageId is null"); requireNonNull(fragment, "fragment is null"); @@ -172,7 +174,7 @@ public static SqlStageExecution createSqlStageExecution( SqlStageExecution sqlStageExecution = new SqlStageExecution( session, - new StageExecutionStateMachine(stageExecutionId, executor, schedulerStats, !fragment.getTableScanSchedulingOrder().isEmpty()), + new StageExecutionStateMachine(stageExecutionId, executor, schedulerStats, !fragment.getTableScanSchedulingOrder().isEmpty(), schedulerSpan), fragment, remoteTaskFactory, nodeTaskMap, @@ -544,7 +546,8 @@ private synchronized RemoteTask scheduleTask(InternalNode node, TaskId taskId, M nodeTaskMap.createTaskStatsTracker(node, taskId), summarizeTaskInfo, tableWriteInfo, - stateMachine); + stateMachine, + stateMachine.getStageSpan()); completeSources.forEach(task::noMoreSplits); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java index ab38b45118b3f..6de2336016a34 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTask.java @@ -17,6 +17,8 @@ import com.facebook.airlift.log.Logger; import com.facebook.airlift.stats.CounterStat; import com.facebook.presto.Session; +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.BufferResult; import com.facebook.presto.execution.buffer.LazyOutputBuffer; @@ -38,9 +40,12 @@ import com.facebook.presto.spi.ConnectorMetadataUpdateHandle; import com.facebook.presto.spi.connector.ConnectorMetadataUpdater; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -51,6 +56,7 @@ import java.net.URI; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -61,6 +67,7 @@ import static com.facebook.presto.execution.TaskState.ABORTED; import static com.facebook.presto.execution.TaskState.FAILED; +import static com.facebook.presto.telemetry.TracingManager.addEvent; import static com.facebook.presto.util.Failures.toFailures; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -89,6 +96,7 @@ public class SqlTask private final AtomicReference taskHolderReference = new AtomicReference<>(new TaskHolder()); private final AtomicBoolean needsPlan = new AtomicBoolean(true); private final long creationTimeInMillis = System.currentTimeMillis(); + private BaseSpan taskSpan = TracingManager.getInvalidSpan(); public static SqlTask createSqlTask( TaskId taskId, @@ -157,48 +165,51 @@ private void initialize(Function onDone, CounterStat failedTasks) { requireNonNull(onDone, "onDone is null"); requireNonNull(failedTasks, "failedTasks is null"); - taskStateMachine.addStateChangeListener(new StateChangeListener() - { - @Override - public void stateChanged(TaskState newState) - { - if (!newState.isDone()) { - return; - } - // Update failed tasks counter - if (newState == FAILED) { - failedTasks.update(1); - } + taskStateMachine.addStateChangeListener(newState -> { + addEvent(taskSpan, "task-state " + newState.name()); + if (!newState.isDone()) { + return; + } - // store final task info - while (true) { - TaskHolder taskHolder = taskHolderReference.get(); - if (taskHolder.isFinished()) { - // another concurrent worker already set the final state - return; - } + // Update failed tasks counter + if (newState == FAILED) { + failedTasks.update(1); + } - if (taskHolderReference.compareAndSet(taskHolder, new TaskHolder(createTaskInfo(taskHolder), taskHolder.getIoStats()))) { - break; - } + // store final task info + while (true) { + TaskHolder taskHolder = taskHolderReference.get(); + if (taskHolder.isFinished()) { + // another concurrent worker already set the final state + return; } - // make sure buffers are cleaned up - if (newState == FAILED || newState == ABORTED) { - // don't close buffers for a failed query - // closed buffers signal to upstream tasks that everything finished cleanly - outputBuffer.fail(); - } - else { - outputBuffer.destroy(); + if (taskHolderReference.compareAndSet(taskHolder, new TaskHolder(createTaskInfo(taskHolder), taskHolder.getIoStats()))) { + break; } + } - try { - onDone.apply(SqlTask.this); - } - catch (Exception e) { - log.warn(e, "Error running task cleanup callback %s", SqlTask.this.taskId); + // make sure buffers are cleaned up + if (newState == FAILED || newState == ABORTED) { + // don't close buffers for a failed query + // closed buffers signal to upstream tasks that everything finished cleanly + outputBuffer.fail(); + } + else { + outputBuffer.destroy(); + } + + try { + onDone.apply(SqlTask.this); + } + catch (Exception e) { + log.warn(e, "Error running task cleanup callback %s", SqlTask.this.taskId); + } + + if (newState.isDone()) { + if (taskSpan != null) { + taskSpan.end(); } } }); @@ -433,7 +444,8 @@ public TaskInfo updateTask( Optional fragment, List sources, OutputBuffers outputBuffers, - Optional tableWriteInfo) + Optional tableWriteInfo, + BaseSpan span) { try { // The LazyOutput buffer does not support write methods, so the actual @@ -453,6 +465,11 @@ public TaskInfo updateTask( if (taskExecution == null) { checkState(fragment.isPresent(), "fragment must be present"); checkState(tableWriteInfo.isPresent(), "tableWriteInfo must be present"); + + if (TelemetryConfig.getTracingEnabled() && Objects.nonNull(taskSpan)) { + taskSpan = TracingManager.getSpan(span, TracingEnum.TASK.getName(), ImmutableMap.of("node id", nodeId, "QUERY_ID", taskId.getQueryId().toString(), "STAGE_ID", taskId.getStageId().toString(), "TASK_ID", taskId.toString(), "task instance id", getTaskInstanceId())); + } + taskExecution = sqlTaskExecutionFactory.create( session, queryContext, @@ -461,7 +478,8 @@ public TaskInfo updateTask( taskExchangeClientManager, fragment.get(), sources, - tableWriteInfo.get()); + tableWriteInfo.get(), + taskSpan); taskHolderReference.compareAndSet(taskHolder, new TaskHolder(taskExecution)); needsPlan.set(false); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java index 43885270d21a6..9042491e39ba4 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecution.java @@ -14,6 +14,7 @@ package com.facebook.presto.execution; import com.facebook.airlift.concurrent.SetThreadName; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.event.SplitMonitor; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.buffer.BufferState; @@ -30,7 +31,9 @@ import com.facebook.presto.spi.SplitWeight; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.StageExecutionDescriptor; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -70,6 +73,7 @@ import static com.facebook.presto.execution.SqlTaskExecution.SplitsState.FINISHED; import static com.facebook.presto.execution.SqlTaskExecution.SplitsState.NO_MORE_SPLITS; import static com.facebook.presto.operator.PipelineExecutionStrategy.UNGROUPED_EXECUTION; +import static com.facebook.presto.telemetry.TracingManager.addEvent; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -130,6 +134,7 @@ public class SqlTaskExecution // guarded for update only @GuardedBy("this") private final ConcurrentMap remoteSources = new ConcurrentHashMap<>(); + private static BaseSpan taskSpan; @GuardedBy("this") private long maxAcknowledgedSplit = Long.MIN_VALUE; @@ -150,7 +155,8 @@ static SqlTaskExecution createSqlTaskExecution( LocalExecutionPlan localExecutionPlan, TaskExecutor taskExecutor, Executor notificationExecutor, - SplitMonitor queryMonitor) + SplitMonitor queryMonitor, + BaseSpan taskSpan) { SqlTaskExecution task = new SqlTaskExecution( taskStateMachine, @@ -159,7 +165,11 @@ static SqlTaskExecution createSqlTaskExecution( localExecutionPlan, taskExecutor, queryMonitor, - notificationExecutor); + notificationExecutor, + taskSpan); + + taskExecutor.setTaskId(task.taskId); + try (SetThreadName ignored = new SetThreadName("Task-%s", task.getTaskId())) { // The scheduleDriversForTaskLifeCycle method calls enqueueDriverSplitRunner, which registers a callback with access to this object. // The call back is accessed from another thread, so this code can not be placed in the constructor. @@ -176,7 +186,8 @@ private SqlTaskExecution( LocalExecutionPlan localExecutionPlan, TaskExecutor taskExecutor, SplitMonitor splitMonitor, - Executor notificationExecutor) + Executor notificationExecutor, + BaseSpan taskSpan) { this.taskStateMachine = requireNonNull(taskStateMachine, "taskStateMachine is null"); this.taskId = taskStateMachine.getTaskId(); @@ -558,6 +569,7 @@ private synchronized void enqueueDriverSplitRunner(boolean forceRunSplit, List() { @@ -570,7 +582,7 @@ public void onSuccess(Object result) checkTaskCompletion(); - splitMonitor.splitCompletedEvent(taskId, getDriverStats()); + splitMonitor.splitCompletedEvent(taskId, getDriverStats(), pipelineSpan); } } @@ -584,7 +596,7 @@ public void onFailure(Throwable cause) status.decrementRemainingDriver(splitRunner.getLifespan()); // fire failed event with cause - splitMonitor.splitFailedEvent(taskId, getDriverStats(), cause); + splitMonitor.splitFailedEvent(taskId, getDriverStats(), cause, pipelineSpan); } } @@ -920,11 +932,15 @@ private class DriverSplitRunnerFactory private final DriverFactory driverFactory; private final PipelineContext pipelineContext; private boolean closed; + private final BaseSpan pipelineSpan; + private final int pipelineId; private DriverSplitRunnerFactory(DriverFactory driverFactory, boolean partitioned) { this.driverFactory = driverFactory; this.pipelineContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver(), partitioned); + this.pipelineId = pipelineContext.getPipelineId(); + this.pipelineSpan = TracingManager.getSpan(taskSpan, TracingEnum.PIPELINE.getName(), ImmutableMap.of("QUERY_ID", taskId.getQueryId().toString(), "STAGE_ID", taskId.getStageId().toString(), "TASK_ID", taskId.toString(), "PIPELINE_ID", taskId.getStageId() + "-" + pipelineContext.getPipelineId())); } // TODO: split this method into two: createPartitionedDriverRunner and createUnpartitionedDriverRunner. @@ -993,6 +1009,7 @@ public void closeDriverFactoryIfFullyCreated() return; } driverFactory.noMoreDrivers(); + addEvent(pipelineSpan, "query_state", "driver-factory-closed"); closed = true; } @@ -1049,6 +1066,12 @@ public Lifespan getLifespan() return lifespan; } + @Override + public BaseSpan getPipelineSpan() + { + return driverSplitRunnerFactory.pipelineSpan; + } + @Override public synchronized boolean isFinished() { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java index 90c930ee6b02f..912159c77dfc0 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskExecutionFactory.java @@ -23,6 +23,7 @@ import com.facebook.presto.memory.QueryContext; import com.facebook.presto.operator.TaskContext; import com.facebook.presto.operator.TaskExchangeClientManager; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.planner.HttpRemoteSourceFactory; import com.facebook.presto.sql.planner.LocalExecutionPlanner; @@ -85,7 +86,8 @@ public SqlTaskExecution create( TaskExchangeClientManager taskExchangeClientManager, PlanFragment fragment, List sources, - TableWriteInfo tableWriteInfo) + TableWriteInfo tableWriteInfo, + BaseSpan taskSpan) { TaskContext taskContext = queryContext.addTaskContext( taskStateMachine, @@ -123,6 +125,7 @@ public SqlTaskExecution create( localExecutionPlan, taskExecutor, taskNotificationExecutor, - splitMonitor); + splitMonitor, + taskSpan); } } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java index 3d127247fd5ee..ded76ef089468 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlTaskManager.java @@ -43,6 +43,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.connector.ConnectorMetadataUpdater; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.spiller.LocalSpillManager; import com.facebook.presto.spiller.NodeSpillConfig; import com.facebook.presto.sql.gen.OrderingCompiler; @@ -399,7 +400,8 @@ public TaskInfo updateTask( Optional fragment, List sources, OutputBuffers outputBuffers, - Optional tableWriteInfo) + Optional tableWriteInfo, + BaseSpan span) { requireNonNull(session, "session is null"); requireNonNull(taskId, "taskId is null"); @@ -431,7 +433,7 @@ public TaskInfo updateTask( queryContext.setHeapDumpFilePath(heapDumpFilePath); sqlTask.recordHeartbeat(); - return sqlTask.updateTask(session, fragment, sources, outputBuffers, tableWriteInfo); + return sqlTask.updateTask(session, fragment, sources, outputBuffers, tableWriteInfo, span); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java index fca02027824e7..3fa063d58e8a2 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java @@ -16,13 +16,18 @@ import com.facebook.airlift.log.Logger; import com.facebook.airlift.stats.Distribution; import com.facebook.presto.common.RuntimeStats; +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.scheduler.ScheduleResult; import com.facebook.presto.execution.scheduler.SplitSchedulerStats; import com.facebook.presto.operator.BlockedReason; import com.facebook.presto.operator.TaskStats; +import com.facebook.presto.spi.telemetry.BaseSpan; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.util.Failures; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.joda.time.DateTime; import javax.annotation.concurrent.ThreadSafe; @@ -60,6 +65,7 @@ import static com.facebook.presto.execution.StageExecutionState.SCHEDULING; import static com.facebook.presto.execution.StageExecutionState.SCHEDULING_SPLITS; import static com.facebook.presto.execution.StageExecutionState.TERMINAL_STAGE_STATES; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -93,12 +99,14 @@ public class StageExecutionStateMachine private final AtomicLong currentTotalMemory = new AtomicLong(); private final RuntimeStats runtimeStats = new RuntimeStats(); + private BaseSpan stageSpan; public StageExecutionStateMachine( StageExecutionId stageExecutionId, ExecutorService executor, SplitSchedulerStats schedulerStats, - boolean containsTableScans) + boolean containsTableScans, + BaseSpan schedulerSpan) { this.stageExecutionId = requireNonNull(stageExecutionId, "stageId is null"); this.scheduledStats = requireNonNull(schedulerStats, "schedulerStats is null"); @@ -108,6 +116,16 @@ public StageExecutionStateMachine( state.addStateChangeListener(state -> log.debug("Stage Execution %s is %s", stageExecutionId, state)); finalInfo = new StateMachine<>("final stage execution " + stageExecutionId, executor, Optional.empty()); + + stageSpan = TracingManager.getSpan(schedulerSpan, TracingEnum.STAGE.getName(), ImmutableMap.of("QUERY_ID", stageExecutionId.getStageId().getQueryId().toString(), "STAGE_ID", stageExecutionId.getStageId().toString())); + + try (BaseSpan spanIgnored = (TelemetryConfig.getTracingEnabled() && (stageSpan != null)) ? scopedSpan(stageSpan) : null) { //Recheck if working + state.addStateChangeListener(state -> { + if ((stageSpan != null)) { + stageSpan.end(); + } + }); + } } public StageExecutionId getStageExecutionId() @@ -120,6 +138,11 @@ public StageExecutionState getState() return state.get(); } + public BaseSpan getStageSpan() + { + return stageSpan; + } + /** * Listener is always notified asynchronously using a dedicated notification thread pool so, care should * be taken to avoid leaking {@code this} when adding a listener in a constructor. Additionally, it is diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java index 3c0a4268ec0c0..7765f0fc12c27 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskId.java @@ -79,6 +79,11 @@ public QueryId getQueryId() return stageExecutionId.getStageId().getQueryId(); } + public StageId getStageId() + { + return stageExecutionId.getStageId(); + } + @Override @JsonValue public String toString() diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java b/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java index 98ff29a00fb25..f57407f401598 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TaskManager.java @@ -22,6 +22,7 @@ import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.MemoryPoolAssignmentsRequest; import com.facebook.presto.metadata.MetadataUpdates; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; @@ -91,7 +92,8 @@ TaskInfo updateTask( Optional fragment, List sources, OutputBuffers outputBuffers, - Optional tableWriteInfo); + Optional tableWriteInfo, + BaseSpan span); /** * Cancels a task. If the task does not already exist, is is created and then diff --git a/presto-main/src/main/java/com/facebook/presto/execution/TrackingRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/TrackingRemoteTaskFactory.java index 033d9ef859b5a..154787f33ac2c 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/TrackingRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/TrackingRemoteTaskFactory.java @@ -21,6 +21,7 @@ import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.Multimap; @@ -50,7 +51,8 @@ public RemoteTask createRemoteTask(Session session, NodeStatsTracker nodeStatsTracker, boolean summarizeTaskInfo, TableWriteInfo tableWriteInfo, - SchedulerStatsTracker schedulerStatsTracker) + SchedulerStatsTracker schedulerStatsTracker, + BaseSpan stageSpan) { RemoteTask task = remoteTaskFactory.createRemoteTask(session, taskId, @@ -61,7 +63,8 @@ public RemoteTask createRemoteTask(Session session, nodeStatsTracker, summarizeTaskInfo, tableWriteInfo, - schedulerStatsTracker); + schedulerStatsTracker, + stageSpan); task.addStateChangeListener(new UpdateQueryStats(stateMachine)); return task; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java b/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java index a2de9b34d1b58..53522884d3f0d 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java @@ -182,6 +182,7 @@ public class TaskExecutor private volatile boolean closed; private volatile boolean lowMemory; + private TaskId taskId; @Inject public TaskExecutor(TaskManagerConfig config, EmbedVersion embedVersion, MultilevelSplitQueue splitQueue) @@ -299,6 +300,11 @@ public TaskExecutor( this.interruptSplitInterval = interruptSplitInterval; } + public void setTaskId(TaskId taskId) + { + this.taskId = taskId; + } + @PostConstruct public synchronized void start() { diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SectionExecutionFactory.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SectionExecutionFactory.java index 26b77ef163ca5..92f5628a7b55b 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SectionExecutionFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SectionExecutionFactory.java @@ -40,6 +40,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.split.SplitSource; import com.facebook.presto.sql.planner.NodePartitionMap; import com.facebook.presto.sql.planner.NodePartitioningManager; @@ -171,7 +172,8 @@ public SectionExecution createSectionExecutions( RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, int attemptId, - CTEMaterializationTracker cteMaterializationTracker) + CTEMaterializationTracker cteMaterializationTracker, + BaseSpan schedulerSpan) { // Only fetch a distribution once per section to ensure all stages see the same machine assignments Map partitioningCache = new HashMap<>(); @@ -188,7 +190,8 @@ public SectionExecution createSectionExecutions( remoteTaskFactory, splitSourceFactory, attemptId, - cteMaterializationTracker); + cteMaterializationTracker, + schedulerSpan); StageExecutionAndScheduler rootStage = getLast(sectionStages); rootStage.getStageExecution().setOutputBuffers(outputBuffers); return new SectionExecution(rootStage, sectionStages); @@ -208,7 +211,8 @@ private List createStreamingLinkedStageExecutions( RemoteTaskFactory remoteTaskFactory, SplitSourceFactory splitSourceFactory, int attemptId, - CTEMaterializationTracker cteMaterializationTracker) + CTEMaterializationTracker cteMaterializationTracker, + BaseSpan schedulerSpan) { ImmutableList.Builder stageExecutionAndSchedulers = ImmutableList.builder(); @@ -224,7 +228,8 @@ private List createStreamingLinkedStageExecutions( executor, failureDetector, schedulerStats, - tableWriteInfo); + tableWriteInfo, + schedulerSpan); PartitioningHandle partitioningHandle = plan.getFragment().getPartitioning(); List remoteSourceNodes = plan.getFragment().getRemoteSourceNodes(); @@ -244,7 +249,8 @@ private List createStreamingLinkedStageExecutions( remoteTaskFactory, splitSourceFactory, attemptId, - cteMaterializationTracker); + cteMaterializationTracker, + schedulerSpan); stageExecutionAndSchedulers.addAll(subTree); childStagesBuilder.add(getLast(subTree).getStageExecution()); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java index a255a939f6ffa..6c5863d04f342 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/SqlQueryScheduler.java @@ -17,6 +17,7 @@ import com.facebook.airlift.log.Logger; import com.facebook.airlift.stats.TimeStat; import com.facebook.presto.Session; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.execution.BasicStageExecutionStats; import com.facebook.presto.execution.LocationFactory; @@ -41,6 +42,7 @@ import com.facebook.presto.spi.plan.PlanFragmentId; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.SplitSourceFactory; @@ -48,7 +50,9 @@ import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.sanity.PlanChecker; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListenableFuture; @@ -65,6 +69,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -151,6 +156,7 @@ public class SqlQueryScheduler private final PartialResultQueryTaskTracker partialResultQueryTaskTracker; private final CTEMaterializationTracker cteMaterializationTracker = new CTEMaterializationTracker(); + private BaseSpan schedulerSpan; public static SqlQueryScheduler createSqlQueryScheduler( LocationFactory locationFactory, @@ -244,6 +250,9 @@ private SqlQueryScheduler( this.sectionedPlan = extractStreamingSections(plan); this.summarizeTaskInfo = summarizeTaskInfo; + BaseSpan querySpan = queryStateMachine.getSession().getQuerySpan(); + this.schedulerSpan = TracingManager.getSpan(querySpan, TracingEnum.SCHEDULER.getName(), ImmutableMap.of("QUERY_ID", queryStateMachine.getQueryId().toString())); + OutputBufferId rootBufferId = getOnlyElement(rootOutputBuffers.getBuffers().keySet()); List stageExecutions = createStageExecutions( sectionExecutionFactory, @@ -276,6 +285,9 @@ else if (state == CANCELED) { // output stage was canceled queryStateMachine.transitionToCanceled(); } + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } }); for (StageExecutionAndScheduler stageExecutionInfo : stageExecutions.values()) { @@ -297,10 +309,16 @@ else if (state == CANCELED) { } if (state == FAILED) { queryStateMachine.transitionToFailed(stageExecution.getStageExecutionInfo().getFailureCause().get().toException()); + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } } else if (state == ABORTED) { // this should never happen, since abort can only be triggered in query clean up after the query is finished queryStateMachine.transitionToFailed(new PrestoException(GENERIC_INTERNAL_ERROR, "Query stage was aborted")); + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } } else if (state == FINISHED) { // checks if there's any new sections available for execution and starts the scheduling if any @@ -377,7 +395,8 @@ private List createStageExecutions( remoteTaskFactory, splitSourceFactory, 0, - cteMaterializationTracker).getSectionStages(); + cteMaterializationTracker, + schedulerSpan).getSectionStages(); stages.addAll(sectionStages); return stages.build(); @@ -555,6 +574,9 @@ else if (!result.getBlocked().isDone()) { catch (Throwable t) { scheduling.set(false); queryStateMachine.transitionToFailed(t); + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } throw t; } finally { @@ -565,6 +587,9 @@ else if (!result.getBlocked().isDone()) { } catch (Throwable t) { queryStateMachine.transitionToFailed(t); + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } // Self-suppression not permitted if (closeError != t) { closeError.addSuppressed(t); @@ -697,7 +722,8 @@ private void updateStageExecutions(StreamingPlanSection section, Map updatedStageExecutions = sectionExecution.getSectionStages().stream() .collect(toImmutableMap(execution -> execution.getStageExecution().getStageExecutionId().getStageId(), identity())); @@ -739,6 +765,9 @@ else if (state == CANCELED) { // output stage was canceled queryStateMachine.transitionToCanceled(); } + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } }); } stageExecution.addStateChangeListener(state -> { @@ -747,10 +776,16 @@ else if (state == CANCELED) { } if (state == FAILED) { queryStateMachine.transitionToFailed(stageExecution.getStageExecutionInfo().getFailureCause().get().toException()); + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } } else if (state == ABORTED) { // this should never happen, since abort can only be triggered in query clean up after the query is finished queryStateMachine.transitionToFailed(new PrestoException(GENERIC_INTERNAL_ERROR, "Query stage was aborted")); + if (!Objects.isNull(schedulerSpan)) { + schedulerSpan.end(); + } } else if (state == FINISHED) { // checks if there's any new sections available for execution and starts the scheduling if any diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java index ead35ca3cc4be..6376b5d5a4cb1 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -20,10 +20,12 @@ import com.facebook.presto.Session; import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.TelemetryConfig; import com.facebook.presto.common.block.BlockEncodingManager; import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.execution.QueryManager; @@ -71,6 +73,7 @@ import com.facebook.presto.spi.statistics.ComputedStatistics; import com.facebook.presto.spi.statistics.TableStatistics; import com.facebook.presto.spi.statistics.TableStatisticsMetadata; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; @@ -130,6 +133,7 @@ import static com.facebook.presto.spi.TableLayoutFilterCoverage.NOT_APPLICABLE; import static com.facebook.presto.spi.analyzer.ViewDefinition.ViewColumn; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -156,6 +160,7 @@ public class MetadataManager private final ConcurrentMap> catalogsByQueryId = new ConcurrentHashMap<>(); private final Set queriesWithRegisteredCallbacks = ConcurrentHashMap.newKeySet(); + private Boolean skipSpan; @VisibleForTesting public MetadataManager( @@ -202,7 +207,7 @@ public MetadataManager( this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionManager is null"); this.procedures = new ProcedureRegistry(functionAndTypeManager); - + this.skipSpan = TelemetryConfig.getSpanSampling(); verifyComparableOrderableContract(); } @@ -302,350 +307,398 @@ public void registerBuiltInFunctions(List functionInfos) @Override public List listSchemaNames(Session session, String catalogName) { - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, catalogName); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_SCHEMA_NAMES.getName(), skipSpan)) { + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, catalogName); - ImmutableSet.Builder schemaNames = ImmutableSet.builder(); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); - ConnectorSession connectorSession = session.toConnectorSession(catalogMetadata.getConnectorId()); - for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - metadata.listSchemaNames(connectorSession).stream() - .map(schema -> schema.toLowerCase(Locale.ENGLISH)) - .forEach(schemaNames::add); + ImmutableSet.Builder schemaNames = ImmutableSet.builder(); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorSession connectorSession = session.toConnectorSession(catalogMetadata.getConnectorId()); + for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + metadata.listSchemaNames(connectorSession).stream() + .map(schema -> schema.toLowerCase(Locale.ENGLISH)) + .forEach(schemaNames::add); + } } + return ImmutableList.copyOf(schemaNames.build()); } - return ImmutableList.copyOf(schemaNames.build()); } @Override public Optional getTableHandleForStatisticsCollection(Session session, QualifiedObjectName table, Map analyzeProperties) { - requireNonNull(table, "table is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_TABLE_HANDLE_FOR_STATISTICS_COLLECTION.getName(), skipSpan)) { + requireNonNull(table, "table is null"); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, table.getCatalogName()); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); - ConnectorId connectorId = catalogMetadata.getConnectorId(session, table); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, table.getCatalogName()); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorId connectorId = catalogMetadata.getConnectorId(session, table); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorTableHandle tableHandle = metadata.getTableHandleForStatisticsCollection(session.toConnectorSession(connectorId), toSchemaTableName(table), analyzeProperties); - if (tableHandle != null) { - return Optional.of(new TableHandle( - connectorId, - tableHandle, - catalogMetadata.getTransactionHandleFor(connectorId), - Optional.empty())); + ConnectorTableHandle tableHandle = metadata.getTableHandleForStatisticsCollection(session.toConnectorSession(connectorId), toSchemaTableName(table), analyzeProperties); + if (tableHandle != null) { + return Optional.of(new TableHandle( + connectorId, + tableHandle, + catalogMetadata.getTransactionHandleFor(connectorId), + Optional.empty())); + } } + return Optional.empty(); } - return Optional.empty(); } @Override public Optional getHandleVersion(Session session, QualifiedObjectName tableName, Optional tableVersion) { - return getOptionalTableHandle(session, transactionManager, tableName, tableVersion); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_HANDLE_VERSION.getName(), skipSpan)) { + return getOptionalTableHandle(session, transactionManager, tableName, tableVersion); + } } @Override public Optional getSystemTable(Session session, QualifiedObjectName tableName) { - requireNonNull(session, "session is null"); - requireNonNull(tableName, "table is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_SYSTEM_TABLE.getName(), skipSpan)) { + requireNonNull(session, "session is null"); + requireNonNull(tableName, "table is null"); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, tableName.getCatalogName()); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, tableName.getCatalogName()); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); - // we query only main connector for runtime system tables - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + // we query only main connector for runtime system tables + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - return metadata.getSystemTable(session.toConnectorSession(connectorId), toSchemaTableName(tableName)); + return metadata.getSystemTable(session.toConnectorSession(connectorId), toSchemaTableName(tableName)); + } + return Optional.empty(); } - return Optional.empty(); } @Override public TableLayoutResult getLayout(Session session, TableHandle table, Constraint constraint, Optional> desiredColumns) { - long startTime = System.nanoTime(); - checkArgument(!constraint.getSummary().isNone(), "Cannot get Layout if constraint is none"); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_LAYOUT.getName(), skipSpan)) { + long startTime = System.nanoTime(); + checkArgument(!constraint.getSummary().isNone(), "Cannot get Layout if constraint is none"); - ConnectorId connectorId = table.getConnectorId(); - ConnectorTableHandle connectorTable = table.getConnectorHandle(); + ConnectorId connectorId = table.getConnectorId(); + ConnectorTableHandle connectorTable = table.getConnectorHandle(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - ConnectorTableLayoutResult layout = metadata.getTableLayoutForConstraint(connectorSession, connectorTable, constraint, desiredColumns); - session.getRuntimeStats().addMetricValue(GET_LAYOUT_TIME_NANOS, NANO, System.nanoTime() - startTime); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + ConnectorTableLayoutResult layout = metadata.getTableLayoutForConstraint(connectorSession, connectorTable, constraint, desiredColumns); + session.getRuntimeStats().addMetricValue(GET_LAYOUT_TIME_NANOS, NANO, System.nanoTime() - startTime); - return new TableLayoutResult(fromConnectorLayout(connectorId, table.getConnectorHandle(), table.getTransaction(), layout.getTableLayout()), layout.getUnenforcedConstraint()); + return new TableLayoutResult(fromConnectorLayout(connectorId, table.getConnectorHandle(), table.getTransaction(), layout.getTableLayout()), layout.getUnenforcedConstraint()); + } } @Override public TableLayout getLayout(Session session, TableHandle handle) { - ConnectorId connectorId = handle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - return fromConnectorLayout(connectorId, handle.getConnectorHandle(), handle.getTransaction(), metadata.getTableLayout(session.toConnectorSession(connectorId), resolveTableLayout(session, handle))); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_LAYOUT.getName(), skipSpan)) { + ConnectorId connectorId = handle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + return fromConnectorLayout(connectorId, handle.getConnectorHandle(), handle.getTransaction(), metadata.getTableLayout(session.toConnectorSession(connectorId), resolveTableLayout(session, handle))); + } } @Override public TableHandle getAlternativeTableHandle(Session session, TableHandle tableHandle, PartitioningHandle partitioningHandle) { - checkArgument(partitioningHandle.getConnectorId().isPresent(), "Expect partitioning handle from connector, got system partitioning handle"); - ConnectorId connectorId = partitioningHandle.getConnectorId().get(); - checkArgument(connectorId.equals(tableHandle.getConnectorId()), "ConnectorId of tableLayoutHandle and partitioningHandle does not match"); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorTableLayoutHandle newTableLayoutHandle = metadata.getAlternativeLayoutHandle(session.toConnectorSession(connectorId), tableHandle.getLayout().get(), partitioningHandle.getConnectorHandle()); - return new TableHandle(tableHandle.getConnectorId(), tableHandle.getConnectorHandle(), tableHandle.getTransaction(), Optional.of(newTableLayoutHandle)); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_ALTERNATE_TABLE_HANDLE.getName(), skipSpan)) { + checkArgument(partitioningHandle.getConnectorId().isPresent(), "Expect partitioning handle from connector, got system partitioning handle"); + ConnectorId connectorId = partitioningHandle.getConnectorId().get(); + checkArgument(connectorId.equals(tableHandle.getConnectorId()), "ConnectorId of tableLayoutHandle and partitioningHandle does not match"); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + ConnectorTableLayoutHandle newTableLayoutHandle = metadata.getAlternativeLayoutHandle(session.toConnectorSession(connectorId), tableHandle.getLayout().get(), partitioningHandle.getConnectorHandle()); + return new TableHandle(tableHandle.getConnectorId(), tableHandle.getConnectorHandle(), tableHandle.getTransaction(), Optional.of(newTableLayoutHandle)); + } } @Override public boolean isLegacyGetLayoutSupported(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); + try (BaseSpan ignored = scopedSpan(TracingEnum.IS_LEGACY_GET_LAYOUT_SUPPORTED.getName(), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - return metadata.isLegacyGetLayoutSupported(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + return metadata.isLegacyGetLayoutSupported(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + } } @Override public Optional getCommonPartitioning(Session session, PartitioningHandle left, PartitioningHandle right) { - Optional leftConnectorId = left.getConnectorId(); - Optional rightConnectorId = right.getConnectorId(); - if (!leftConnectorId.isPresent() || !rightConnectorId.isPresent() || !leftConnectorId.equals(rightConnectorId)) { - return Optional.empty(); - } - if (!left.getTransactionHandle().equals(right.getTransactionHandle())) { - return Optional.empty(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_COMMON_PARTITIONING.getName(), skipSpan)) { + Optional leftConnectorId = left.getConnectorId(); + Optional rightConnectorId = right.getConnectorId(); + if (!leftConnectorId.isPresent() || !rightConnectorId.isPresent() || !leftConnectorId.equals(rightConnectorId)) { + return Optional.empty(); + } + if (!left.getTransactionHandle().equals(right.getTransactionHandle())) { + return Optional.empty(); + } + ConnectorId connectorId = leftConnectorId.get(); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + Optional commonHandle = metadata.getCommonPartitioningHandle(session.toConnectorSession(connectorId), left.getConnectorHandle(), right.getConnectorHandle()); + return commonHandle.map(handle -> new PartitioningHandle(Optional.of(connectorId), left.getTransactionHandle(), handle)); } - ConnectorId connectorId = leftConnectorId.get(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - Optional commonHandle = metadata.getCommonPartitioningHandle(session.toConnectorSession(connectorId), left.getConnectorHandle(), right.getConnectorHandle()); - return commonHandle.map(handle -> new PartitioningHandle(Optional.of(connectorId), left.getTransactionHandle(), handle)); } @Override public boolean isRefinedPartitioningOver(Session session, PartitioningHandle left, PartitioningHandle right) { - Optional leftConnectorId = left.getConnectorId(); - Optional rightConnectorId = right.getConnectorId(); - if (!leftConnectorId.isPresent() || !rightConnectorId.isPresent() || !leftConnectorId.equals(rightConnectorId)) { - return false; - } - if (!left.getTransactionHandle().equals(right.getTransactionHandle())) { - return false; - } - ConnectorId connectorId = leftConnectorId.get(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.IS_REFINED_PARTITIONING_OVER.getName(), skipSpan)) { + Optional leftConnectorId = left.getConnectorId(); + Optional rightConnectorId = right.getConnectorId(); + if (!leftConnectorId.isPresent() || !rightConnectorId.isPresent() || !leftConnectorId.equals(rightConnectorId)) { + return false; + } + if (!left.getTransactionHandle().equals(right.getTransactionHandle())) { + return false; + } + ConnectorId connectorId = leftConnectorId.get(); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - return metadata.isRefinedPartitioningOver(session.toConnectorSession(connectorId), left.getConnectorHandle(), right.getConnectorHandle()); + return metadata.isRefinedPartitioningOver(session.toConnectorSession(connectorId), left.getConnectorHandle(), right.getConnectorHandle()); + } } @Override public PartitioningHandle getPartitioningHandleForExchange(Session session, String catalogName, int partitionCount, List partitionTypes) { - CatalogMetadata catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalogName) - .orElseThrow(() -> new PrestoException(NOT_FOUND, format("Catalog '%s' does not exist", catalogName))); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorPartitioningHandle connectorPartitioningHandle = metadata.getPartitioningHandleForExchange(session.toConnectorSession(connectorId), partitionCount, partitionTypes); - ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); - return new PartitioningHandle(Optional.of(connectorId), Optional.of(transaction), connectorPartitioningHandle); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_PARTITIONING_HANDLE.getName(), skipSpan)) { + CatalogMetadata catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalogName) + .orElseThrow(() -> new PrestoException(NOT_FOUND, format("Catalog '%s' does not exist", catalogName))); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + ConnectorPartitioningHandle connectorPartitioningHandle = metadata.getPartitioningHandleForExchange(session.toConnectorSession(connectorId), partitionCount, partitionTypes); + ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); + return new PartitioningHandle(Optional.of(connectorId), Optional.of(transaction), connectorPartitioningHandle); + } } @Override public Optional getInfo(Session session, TableHandle handle) { - ConnectorId connectorId = handle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return handle.getLayout().flatMap(tableLayout -> metadata.getInfo(tableLayout)); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_INFO.getName(), skipSpan)) { + ConnectorId connectorId = handle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return handle.getLayout().flatMap(tableLayout -> metadata.getInfo(tableLayout)); + } } @Override public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); - if (tableMetadata.getColumns().isEmpty()) { - throw new PrestoException(NOT_SUPPORTED, "Table has no columns: " + tableHandle); - } + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_TABLE_METADATA.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + if (tableMetadata.getColumns().isEmpty()) { + throw new PrestoException(NOT_SUPPORTED, "Table has no columns: " + tableHandle); + } - return new TableMetadata(connectorId, tableMetadata); + return new TableMetadata(connectorId, tableMetadata); + } } @Override public TableStatistics getTableStatistics(Session session, TableHandle tableHandle, List columnHandles, Constraint constraint) { - try { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.getTableStatistics(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableHandle.getLayout(), columnHandles, constraint); - } - catch (RuntimeException e) { - if (isIgnoreStatsCalculatorFailures(session)) { - log.error(e, "Error occurred when computing stats for query %s", session.getQueryId()); - return TableStatistics.empty(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_TABLE_STATISTICS.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + try { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.getTableStatistics(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableHandle.getLayout(), columnHandles, constraint); + } + catch (RuntimeException e) { + if (isIgnoreStatsCalculatorFailures(session)) { + log.error(e, "Error occurred when computing stats for query %s", session.getQueryId()); + return TableStatistics.empty(); + } + throw e; } - throw e; } } @Override public Map getColumnHandles(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - Map handles = metadata.getColumnHandles(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_COLUMN_HANDLES.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + Map handles = metadata.getColumnHandles(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); - ImmutableMap.Builder map = ImmutableMap.builder(); - for (Entry mapEntry : handles.entrySet()) { - map.put(mapEntry.getKey().toLowerCase(ENGLISH), mapEntry.getValue()); + ImmutableMap.Builder map = ImmutableMap.builder(); + for (Entry mapEntry : handles.entrySet()) { + map.put(mapEntry.getKey().toLowerCase(ENGLISH), mapEntry.getValue()); + } + return map.build(); } - return map.build(); } @Override public ColumnMetadata getColumnMetadata(Session session, TableHandle tableHandle, ColumnHandle columnHandle) { - requireNonNull(tableHandle, "tableHandle is null"); - requireNonNull(columnHandle, "columnHandle is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_COLUMN_METADATA.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + requireNonNull(tableHandle, "tableHandle is null"); + requireNonNull(columnHandle, "columnHandle is null"); - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.getColumnMetadata(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), columnHandle); + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.getColumnMetadata(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), columnHandle); + } } @Override public TupleDomain toExplainIOConstraints(Session session, TableHandle tableHandle, TupleDomain constraints) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.EXPLAIN_IO_CONSTRAINTS.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.toExplainIOConstraints(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), constraints); + return metadata.toExplainIOConstraints(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), constraints); + } } @Override public List listTables(Session session, QualifiedTablePrefix prefix) { - requireNonNull(prefix, "prefix is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_TABLES.getName(), ImmutableMap.of("CATALOG", prefix.getCatalogName(), "SCHEMA", prefix.getSchemaName().orElse(""), "TABLE", prefix.getTableName().orElse("")), skipSpan)) { + requireNonNull(prefix, "prefix is null"); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); - Set tables = new LinkedHashSet<>(); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); + Set tables = new LinkedHashSet<>(); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); - for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - metadata.listTables(connectorSession, prefix.getSchemaName()).stream() - .map(convertFromSchemaTableName(prefix.getCatalogName())) - .filter(prefix::matches) - .forEach(tables::add); + for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + metadata.listTables(connectorSession, prefix.getSchemaName()).stream() + .map(convertFromSchemaTableName(prefix.getCatalogName())) + .filter(prefix::matches) + .forEach(tables::add); + } } + return ImmutableList.copyOf(tables); } - return ImmutableList.copyOf(tables); } @Override public Map> listTableColumns(Session session, QualifiedTablePrefix prefix) { - requireNonNull(prefix, "prefix is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_TABLE_COLUMNS.getName(), ImmutableMap.of("CATALOG", prefix.getCatalogName(), "SCHEMA", prefix.getSchemaName().orElse(""), "TABLE", prefix.getTableName().orElse("")), skipSpan)) { + requireNonNull(prefix, "prefix is null"); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); - Map> tableColumns = new HashMap<>(); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); + Map> tableColumns = new HashMap<>(); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); - SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix(); - for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix(); + for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - for (Entry> entry : metadata.listTableColumns(connectorSession, tablePrefix).entrySet()) { - QualifiedObjectName tableName = new QualifiedObjectName( - prefix.getCatalogName(), - entry.getKey().getSchemaName(), - entry.getKey().getTableName()); - tableColumns.put(tableName, entry.getValue()); - } + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + for (Entry> entry : metadata.listTableColumns(connectorSession, tablePrefix).entrySet()) { + QualifiedObjectName tableName = new QualifiedObjectName( + prefix.getCatalogName(), + entry.getKey().getSchemaName(), + entry.getKey().getTableName()); + tableColumns.put(tableName, entry.getValue()); + } - // if table and view names overlap, the view wins - for (Entry entry : metadata.getViews(connectorSession, tablePrefix).entrySet()) { - QualifiedObjectName tableName = new QualifiedObjectName( - prefix.getCatalogName(), - entry.getKey().getSchemaName(), - entry.getKey().getTableName()); + // if table and view names overlap, the view wins + for (Entry entry : metadata.getViews(connectorSession, tablePrefix).entrySet()) { + QualifiedObjectName tableName = new QualifiedObjectName( + prefix.getCatalogName(), + entry.getKey().getSchemaName(), + entry.getKey().getTableName()); - ImmutableList.Builder columns = ImmutableList.builder(); - for (ViewColumn column : deserializeView(entry.getValue().getViewData()).getColumns()) { - columns.add(new ColumnMetadata(column.getName(), column.getType())); - } + ImmutableList.Builder columns = ImmutableList.builder(); + for (ViewColumn column : deserializeView(entry.getValue().getViewData()).getColumns()) { + columns.add(new ColumnMetadata(column.getName(), column.getType())); + } - tableColumns.put(tableName, columns.build()); + tableColumns.put(tableName, columns.build()); + } } } + return ImmutableMap.copyOf(tableColumns); } - return ImmutableMap.copyOf(tableColumns); } @Override public void createSchema(Session session, CatalogSchemaName schema, Map properties) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, schema.getCatalogName()); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.createSchema(session.toConnectorSession(connectorId), schema.getSchemaName(), properties); + try (BaseSpan ignored = scopedSpan(TracingEnum.CREATE_SCHEMA.getName(), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, schema.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + metadata.createSchema(session.toConnectorSession(connectorId), schema.getSchemaName(), properties); + } } @Override public void dropSchema(Session session, CatalogSchemaName schema) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, schema.getCatalogName()); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.dropSchema(session.toConnectorSession(connectorId), schema.getSchemaName()); + try (BaseSpan ignored = scopedSpan(TracingEnum.DROP_SCHEMA.getName(), ImmutableMap.of("CATALOG", schema.getCatalogName(), "SCHEMA", schema.getSchemaName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, schema.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + metadata.dropSchema(session.toConnectorSession(connectorId), schema.getSchemaName()); + } } @Override public void renameSchema(Session session, CatalogSchemaName source, String target) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, source.getCatalogName()); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.renameSchema(session.toConnectorSession(connectorId), source.getSchemaName(), target); + try (BaseSpan ignored = scopedSpan(TracingEnum.RENAME_SCHEMA.getName(), ImmutableMap.of("CATALOG", source.getCatalogName(), "SCHEMA", source.getSchemaName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, source.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + metadata.renameSchema(session.toConnectorSession(connectorId), source.getSchemaName(), target); + } } @Override public void createTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.createTable(session.toConnectorSession(connectorId), tableMetadata, ignoreExisting); + try (BaseSpan ignored = scopedSpan(TracingEnum.CREATE_TABLE.getName(), ImmutableMap.of("CATALOG", catalogName, "SCHEMA", tableMetadata.getTable().getSchemaName(), "TABLE", tableMetadata.getTable().getTableName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + metadata.createTable(session.toConnectorSession(connectorId), tableMetadata, ignoreExisting); + } } @Override public TableHandle createTemporaryTable(Session session, String catalogName, List columns, Optional partitioningMetadata) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorTableHandle connectorTableHandle = metadata.createTemporaryTable( - session.toConnectorSession(connectorId), - columns, - partitioningMetadata.map(partitioning -> createConnectorPartitioningMetadata(connectorId, partitioning))); - return new TableHandle(connectorId, connectorTableHandle, catalogMetadata.getTransactionHandleFor(connectorId), Optional.empty()); + try (BaseSpan ignored = scopedSpan(TracingEnum.CREATE_TEMPORARY_TABLE.getName(), ImmutableMap.of("CATALOG", catalogName), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorTableHandle connectorTableHandle = metadata.createTemporaryTable( + session.toConnectorSession(connectorId), + columns, + partitioningMetadata.map(partitioning -> createConnectorPartitioningMetadata(connectorId, partitioning))); + return new TableHandle(connectorId, connectorTableHandle, catalogMetadata.getTransactionHandleFor(connectorId), Optional.empty()); + } } private static ConnectorPartitioningMetadata createConnectorPartitioningMetadata(ConnectorId connectorId, PartitioningMetadata partitioningMetadata) @@ -663,15 +716,17 @@ private static ConnectorPartitioningMetadata createConnectorPartitioningMetadata @Override public void renameTable(Session session, TableHandle tableHandle, QualifiedObjectName newTableName) { - String catalogName = newTableName.getCatalogName(); - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - if (!tableHandle.getConnectorId().equals(connectorId)) { - throw new PrestoException(SYNTAX_ERROR, "Cannot rename tables across catalogs"); - } + try (BaseSpan ignored = scopedSpan(TracingEnum.RENAME_TABLE.getName(), ImmutableMap.of("CATALOG", newTableName.getCatalogName(), "SCHEMA", newTableName.getSchemaName(), "TABLE", newTableName.getObjectName()), skipSpan)) { + String catalogName = newTableName.getCatalogName(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + if (!tableHandle.getConnectorId().equals(connectorId)) { + throw new PrestoException(SYNTAX_ERROR, "Cannot rename tables across catalogs"); + } - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.renameTable(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), toSchemaTableName(newTableName)); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + metadata.renameTable(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), toSchemaTableName(newTableName)); + } } @Override @@ -685,323 +740,385 @@ public void setTableProperties(Session session, TableHandle tableHandle, Map getInsertLayout(Session session, TableHandle table) { - ConnectorId connectorId = table.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_INSERT_LAYOUT.getName(), ImmutableMap.of("HANDLE", table.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = table.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - return metadata.getInsertLayout(session.toConnectorSession(connectorId), table.getConnectorHandle()) - .map(layout -> new NewTableLayout(connectorId, catalogMetadata.getTransactionHandleFor(connectorId), layout)); + return metadata.getInsertLayout(session.toConnectorSession(connectorId), table.getConnectorHandle()) + .map(layout -> new NewTableLayout(connectorId, catalogMetadata.getTransactionHandleFor(connectorId), layout)); + } } @Override public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - return metadata.getStatisticsCollectionMetadataForWrite(session.toConnectorSession(connectorId), tableMetadata); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_STATISTICS_COLLECTION_METADATA_FOR_WRITE.getName(), ImmutableMap.of("CATALOG", catalogName, "SCHEMA", tableMetadata.getTable().getSchemaName(), "TABLE", tableMetadata.getTable().getTableName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + return metadata.getStatisticsCollectionMetadataForWrite(session.toConnectorSession(connectorId), tableMetadata); + } } @Override public TableStatisticsMetadata getStatisticsCollectionMetadata(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - return metadata.getStatisticsCollectionMetadata(session.toConnectorSession(connectorId), tableMetadata); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_STATISTICS_COLLECTION_METADATA.getName(), ImmutableMap.of("CATALOG", catalogName, "SCHEMA", tableMetadata.getTable().getSchemaName(), "TABLE", tableMetadata.getTable().getTableName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + return metadata.getStatisticsCollectionMetadata(session.toConnectorSession(connectorId), tableMetadata); + } } @Override public AnalyzeTableHandle beginStatisticsCollection(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.BEGIN_STATISTICS_COLLECTION.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); - ConnectorTableHandle connectorTableHandle = metadata.beginStatisticsCollection(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); - return new AnalyzeTableHandle(connectorId, transactionHandle, connectorTableHandle); + ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); + ConnectorTableHandle connectorTableHandle = metadata.beginStatisticsCollection(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + return new AnalyzeTableHandle(connectorId, transactionHandle, connectorTableHandle); + } } @Override public void finishStatisticsCollection(Session session, AnalyzeTableHandle tableHandle, Collection computedStatistics) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); - catalogMetadata.getMetadata().finishStatisticsCollection(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), computedStatistics); + try (BaseSpan ignored = scopedSpan(TracingEnum.FINISH_STATISTICS_COLLECTION.getName(), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + catalogMetadata.getMetadata().finishStatisticsCollection(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), computedStatistics); + } } @Override public Optional getNewTableLayout(Session session, String catalogName, ConnectorTableMetadata tableMetadata) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_STATISTICS_COLLECTION_METADATA.getName(), ImmutableMap.of("CATALOG", catalogName, "SCHEMA", tableMetadata.getTable().getSchemaName(), "TABLE", tableMetadata.getTable().getTableName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - return metadata.getNewTableLayout(connectorSession, tableMetadata) - .map(layout -> new NewTableLayout(connectorId, transactionHandle, layout)); + ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + return metadata.getNewTableLayout(connectorSession, tableMetadata) + .map(layout -> new NewTableLayout(connectorId, transactionHandle, layout)); + } } @Override public void beginQuery(Session session, Set connectors) { - for (ConnectorId connectorId : connectors) { - ConnectorMetadata metadata = getMetadata(session, connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - metadata.beginQuery(connectorSession); - registerCatalogForQueryId(session.getQueryId(), metadata); + try (BaseSpan ignored = scopedSpan(TracingEnum.BEGIN_QUERY.getName(), skipSpan)) { + for (ConnectorId connectorId : connectors) { + ConnectorMetadata metadata = getMetadata(session, connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + metadata.beginQuery(connectorSession); + registerCatalogForQueryId(session.getQueryId(), metadata); + } } } private void registerCatalogForQueryId(QueryId queryId, ConnectorMetadata metadata) { - catalogsByQueryId.putIfAbsent(queryId.getId(), new ArrayList<>()); - catalogsByQueryId.get(queryId.getId()).add(metadata); + try (BaseSpan ignored = scopedSpan(TracingEnum.REGISTER_CATALOG_FOR_QUERY_ID.getName(), skipSpan)) { + catalogsByQueryId.putIfAbsent(queryId.getId(), new ArrayList<>()); + catalogsByQueryId.get(queryId.getId()).add(metadata); + } } @Override public void cleanupQuery(Session session) { - try { - Collection catalogs = catalogsByQueryId.get(session.getQueryId().getId()); - if (catalogs == null) { - return; - } + try (BaseSpan ignored = scopedSpan(TracingEnum.CLEAN_UP_QUERY.getName(), skipSpan)) { + try { + Collection catalogs = catalogsByQueryId.get(session.getQueryId().getId()); + if (catalogs == null) { + return; + } - for (ConnectorMetadata metadata : catalogs) { - metadata.cleanupQuery(session.toConnectorSession()); + for (ConnectorMetadata metadata : catalogs) { + metadata.cleanupQuery(session.toConnectorSession()); + } + } + finally { + catalogsByQueryId.remove(session.getQueryId().getId()); } - } - finally { - catalogsByQueryId.remove(session.getQueryId().getId()); } } @Override public OutputTableHandle beginCreateTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, Optional layout) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.BEGIN_CREATE_TABLE.getName(), ImmutableMap.of("CATALOG", catalogName, "SCHEMA", tableMetadata.getTable().getSchemaName(), "TABLE", tableMetadata.getTable().getTableName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - ConnectorOutputTableHandle handle = metadata.beginCreateTable(connectorSession, tableMetadata, layout.map(NewTableLayout::getLayout)); - return new OutputTableHandle(connectorId, transactionHandle, handle); + ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + ConnectorOutputTableHandle handle = metadata.beginCreateTable(connectorSession, tableMetadata, layout.map(NewTableLayout::getLayout)); + return new OutputTableHandle(connectorId, transactionHandle, handle); + } } @Override public Optional finishCreateTable(Session session, OutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.finishCreateTable(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments, computedStatistics); + try (BaseSpan ignored = scopedSpan(TracingEnum.FINISH_CREATE_TABLE.getName(), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.finishCreateTable(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments, computedStatistics); + } } @Override public InsertTableHandle beginInsert(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); - ConnectorInsertTableHandle handle = metadata.beginInsert(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); - return new InsertTableHandle(tableHandle.getConnectorId(), transactionHandle, handle); + try (BaseSpan ignored = scopedSpan(TracingEnum.BEGIN_INSERT.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); + ConnectorInsertTableHandle handle = metadata.beginInsert(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + return new InsertTableHandle(tableHandle.getConnectorId(), transactionHandle, handle); + } } @Override public Optional finishInsert(Session session, InsertTableHandle tableHandle, Collection fragments, Collection computedStatistics) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.finishInsert(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments, computedStatistics); + try (BaseSpan ignored = scopedSpan(TracingEnum.FINISH_INSERT.getName(), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.finishInsert(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments, computedStatistics); + } } @Override public ColumnHandle getDeleteRowIdColumnHandle(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.getDeleteRowIdColumnHandle(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_DELETE_ROW_ID_COLUMN_HANDLE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.getDeleteRowIdColumnHandle(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + } } @Override public ColumnHandle getUpdateRowIdColumnHandle(Session session, TableHandle tableHandle, List updatedColumns) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.getUpdateRowIdColumnHandle(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), updatedColumns); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_UPDATE_ROW_ID_COLUMN_HANDLE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.getUpdateRowIdColumnHandle(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), updatedColumns); + } } @Override public boolean supportsMetadataDelete(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.supportsMetadataDelete( - session.toConnectorSession(connectorId), - tableHandle.getConnectorHandle(), - tableHandle.getLayout()); + try (BaseSpan ignored = scopedSpan(TracingEnum.SUPPORTS_METADATA_DELETE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.supportsMetadataDelete( + session.toConnectorSession(connectorId), + tableHandle.getConnectorHandle(), + tableHandle.getLayout()); + } } @Override public OptionalLong metadataDelete(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); - return metadata.metadataDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableHandle.getLayout().get()); + try (BaseSpan ignored = scopedSpan(TracingEnum.METADATA_DELETE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); + return metadata.metadataDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableHandle.getLayout().get()); + } } @Override public TableHandle beginDelete(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); - ConnectorTableHandle newHandle = catalogMetadata.getMetadata().beginDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); - return new TableHandle( - tableHandle.getConnectorId(), - newHandle, - tableHandle.getTransaction(), - Optional.empty()); + try (BaseSpan ignored = scopedSpan(TracingEnum.BEGIN_DELETE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + ConnectorTableHandle newHandle = catalogMetadata.getMetadata().beginDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + return new TableHandle( + tableHandle.getConnectorId(), + newHandle, + tableHandle.getTransaction(), + Optional.empty()); + } } @Override public void finishDelete(Session session, TableHandle tableHandle, Collection fragments) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - metadata.finishDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments); + try (BaseSpan ignored = scopedSpan(TracingEnum.FINISH_DELETE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + metadata.finishDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments); + } } @Override public TableHandle beginUpdate(Session session, TableHandle tableHandle, List updatedColumns) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); - ConnectorTableHandle newHandle = metadata.beginUpdate(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), updatedColumns); - return new TableHandle(tableHandle.getConnectorId(), newHandle, tableHandle.getTransaction(), tableHandle.getLayout()); + try (BaseSpan ignored = scopedSpan(TracingEnum.BEGIN_UPDATE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); + ConnectorTableHandle newHandle = metadata.beginUpdate(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), updatedColumns); + return new TableHandle(tableHandle.getConnectorId(), newHandle, tableHandle.getTransaction(), tableHandle.getLayout()); + } } @Override public void finishUpdate(Session session, TableHandle tableHandle, Collection fragments) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - metadata.finishUpdate(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments); + try (BaseSpan ignored = scopedSpan(TracingEnum.FINISH_UPDATE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + metadata.finishUpdate(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments); + } } @Override public Optional getCatalogHandle(Session session, String catalogName) { - return transactionManager.getOptionalCatalogMetadata(session.getRequiredTransactionId(), catalogName).map(CatalogMetadata::getConnectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_CATALOG_HANDLE.getName(), ImmutableMap.of("CATALOG", catalogName), skipSpan)) { + return transactionManager.getOptionalCatalogMetadata(session.getRequiredTransactionId(), catalogName).map(CatalogMetadata::getConnectorId); + } } @Override public Map getCatalogNames(Session session) { - return transactionManager.getCatalogNames(session.getRequiredTransactionId()); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_CATALOG_NAMES.getName(), skipSpan)) { + return transactionManager.getCatalogNames(session.getRequiredTransactionId()); + } } @Override public List listViews(Session session, QualifiedTablePrefix prefix) { - requireNonNull(prefix, "prefix is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_VIEWS.getName(), ImmutableMap.of("CATALOG", prefix.getCatalogName(), "SCHEMA", prefix.getSchemaName().orElse(""), "TABLE", prefix.getTableName().orElse("")), skipSpan)) { + requireNonNull(prefix, "prefix is null"); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); - Set views = new LinkedHashSet<>(); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); + Set views = new LinkedHashSet<>(); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); - for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - metadata.listViews(connectorSession, prefix.getSchemaName()).stream() - .map(convertFromSchemaTableName(prefix.getCatalogName())) - .filter(prefix::matches) - .forEach(views::add); + for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + metadata.listViews(connectorSession, prefix.getSchemaName()).stream() + .map(convertFromSchemaTableName(prefix.getCatalogName())) + .filter(prefix::matches) + .forEach(views::add); + } } + return ImmutableList.copyOf(views); } - return ImmutableList.copyOf(views); } @Override public Map getViews(Session session, QualifiedTablePrefix prefix) { - requireNonNull(prefix, "prefix is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_VIEWS.getName(), ImmutableMap.of("CATALOG", prefix.getCatalogName(), "SCHEMA", prefix.getSchemaName().orElse(""), "TABLE", prefix.getTableName().orElse("")), skipSpan)) { + requireNonNull(prefix, "prefix is null"); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); - Map views = new LinkedHashMap<>(); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); + Map views = new LinkedHashMap<>(); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); - SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix(); - for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - for (Entry entry : metadata.getViews(connectorSession, tablePrefix).entrySet()) { - QualifiedObjectName viewName = new QualifiedObjectName( - prefix.getCatalogName(), - entry.getKey().getSchemaName(), - entry.getKey().getTableName()); - views.put(viewName, deserializeView(entry.getValue().getViewData())); + SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix(); + for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + for (Entry entry : metadata.getViews(connectorSession, tablePrefix).entrySet()) { + QualifiedObjectName viewName = new QualifiedObjectName( + prefix.getCatalogName(), + entry.getKey().getSchemaName(), + entry.getKey().getTableName()); + views.put(viewName, deserializeView(entry.getValue().getViewData())); + } } } + return ImmutableMap.copyOf(views); } - return ImmutableMap.copyOf(views); } @Override public void createView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, String viewData, boolean replace) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.CREATE_VIEW.getName(), ImmutableMap.of("CATALOG", catalogName, "SCHEMA", viewMetadata.getTable().getSchemaName(), "TABLE", viewMetadata.getTable().getTableName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.createView(session.toConnectorSession(connectorId), viewMetadata, viewData, replace); + metadata.createView(session.toConnectorSession(connectorId), viewMetadata, viewData, replace); + } } @Override @@ -1017,269 +1134,313 @@ public void renameView(Session session, QualifiedObjectName source, QualifiedObj @Override public void dropView(Session session, QualifiedObjectName viewName) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, viewName.getCatalogName()); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.DROP_VIEW.getName(), ImmutableMap.of("CATALOG", viewName.getCatalogName(), "SCHEMA", viewName.getSchemaName(), "TABLE", viewName.getObjectName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, viewName.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.dropView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + metadata.dropView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + } } @Override public void createMaterializedView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, MaterializedViewDefinition viewDefinition, boolean ignoreExisting) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.CREATE_MATERIALIZED_VIEW.getName(), ImmutableMap.of("CATALOG", catalogName, "SCHEMA", viewMetadata != null ? viewMetadata.getTable().getSchemaName() : "", "TABLE", viewMetadata != null ? viewMetadata.getTable().getTableName() : ""), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogName); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.createMaterializedView(session.toConnectorSession(connectorId), viewMetadata, viewDefinition, ignoreExisting); + metadata.createMaterializedView(session.toConnectorSession(connectorId), viewMetadata, viewDefinition, ignoreExisting); + } } @Override public void dropMaterializedView(Session session, QualifiedObjectName viewName) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, viewName.getCatalogName()); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.DROP_MATERIALIZED_VIEW.getName(), ImmutableMap.of("CATALOG", viewName.getCatalogName(), "SCHEMA", viewName.getSchemaName(), "TABLE", viewName.getObjectName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, viewName.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.dropMaterializedView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + metadata.dropMaterializedView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + } } private MaterializedViewStatus getMaterializedViewStatus(Session session, QualifiedObjectName materializedViewName, TupleDomain baseQueryDomain) { - Optional materializedViewHandle = getOptionalTableHandle(session, transactionManager, materializedViewName, Optional.empty()); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_MATERIALIZED_VIEW_STATUS.getName(), ImmutableMap.of("CATALOG", materializedViewName.getCatalogName(), "SCHEMA", materializedViewName.getSchemaName(), "TABLE", materializedViewName.getObjectName()), skipSpan)) { + Optional materializedViewHandle = getOptionalTableHandle(session, transactionManager, materializedViewName, Optional.empty()); - ConnectorId connectorId = materializedViewHandle.get().getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); + ConnectorId connectorId = materializedViewHandle.get().getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); - return session.getRuntimeStats().recordWallTime( - GET_MATERIALIZED_VIEW_STATUS_TIME_NANOS, - () -> metadata.getMaterializedViewStatus(session.toConnectorSession(connectorId), toSchemaTableName(materializedViewName), baseQueryDomain)); + return session.getRuntimeStats().recordWallTime( + GET_MATERIALIZED_VIEW_STATUS_TIME_NANOS, + () -> metadata.getMaterializedViewStatus(session.toConnectorSession(connectorId), toSchemaTableName(materializedViewName), baseQueryDomain)); + } } @Override public InsertTableHandle beginRefreshMaterializedView(Session session, TableHandle tableHandle) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); - ConnectorInsertTableHandle handle = metadata.beginRefreshMaterializedView(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); - return new InsertTableHandle(tableHandle.getConnectorId(), transactionHandle, handle); + try (BaseSpan ignored = scopedSpan(TracingEnum.BEGIN_REFRESH_MATERIALIZED_VIEW.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorTransactionHandle transactionHandle = catalogMetadata.getTransactionHandleFor(connectorId); + ConnectorInsertTableHandle handle = metadata.beginRefreshMaterializedView(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle()); + return new InsertTableHandle(tableHandle.getConnectorId(), transactionHandle, handle); + } } @Override public Optional finishRefreshMaterializedView(Session session, InsertTableHandle tableHandle, Collection fragments, Collection computedStatistics) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadata(session, connectorId); - return metadata.finishRefreshMaterializedView(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments, computedStatistics); + try (BaseSpan ignored = scopedSpan(TracingEnum.FINISH_REFRESH_MATERIALIZED_VIEW.getName(), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadata(session, connectorId); + return metadata.finishRefreshMaterializedView(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments, computedStatistics); + } } @Override public List getReferencedMaterializedViews(Session session, QualifiedObjectName tableName) { - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_REFERENCED_MATERIALIZED_VIEWS.getName(), ImmutableMap.of("CATALOG", tableName.getCatalogName(), "SCHEMA", tableName.getSchemaName(), "TABLE", tableName.getObjectName()), skipSpan)) { + requireNonNull(tableName, "tableName is null"); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, tableName.getCatalogName()); - if (catalog.isPresent()) { - ConnectorMetadata metadata = catalog.get().getMetadata(); - ConnectorSession connectorSession = session.toConnectorSession(catalog.get().getConnectorId()); - Optional> materializedViews = metadata.getReferencedMaterializedViews(connectorSession, toSchemaTableName(tableName)); - if (materializedViews.isPresent()) { - return materializedViews.get().stream().map(convertFromSchemaTableName(tableName.getCatalogName())).collect(toImmutableList()); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, tableName.getCatalogName()); + if (catalog.isPresent()) { + ConnectorMetadata metadata = catalog.get().getMetadata(); + ConnectorSession connectorSession = session.toConnectorSession(catalog.get().getConnectorId()); + Optional> materializedViews = metadata.getReferencedMaterializedViews(connectorSession, toSchemaTableName(tableName)); + if (materializedViews.isPresent()) { + return materializedViews.get().stream().map(convertFromSchemaTableName(tableName.getCatalogName())).collect(toImmutableList()); + } } + return ImmutableList.of(); } - return ImmutableList.of(); } @Override public Optional resolveIndex(Session session, TableHandle tableHandle, Set indexableColumns, Set outputColumns, TupleDomain tupleDomain) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - Optional resolvedIndex = metadata.resolveIndex(connectorSession, tableHandle.getConnectorHandle(), indexableColumns, outputColumns, tupleDomain); - return resolvedIndex.map(resolved -> new ResolvedIndex(tableHandle.getConnectorId(), transaction, resolved)); + try (BaseSpan ignored = scopedSpan(TracingEnum.RESOLVE_INDEX.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(connectorId); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + Optional resolvedIndex = metadata.resolveIndex(connectorSession, tableHandle.getConnectorHandle(), indexableColumns, outputColumns, tupleDomain); + return resolvedIndex.map(resolved -> new ResolvedIndex(tableHandle.getConnectorId(), transaction, resolved)); + } } @Override public void createRole(Session session, String role, Optional grantor, String catalog) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.CREATE_ROLE.getName(), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.createRole(session.toConnectorSession(connectorId), role, grantor); + metadata.createRole(session.toConnectorSession(connectorId), role, grantor); + } } @Override public void dropRole(Session session, String role, String catalog) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.DROP_ROLE.getName(), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.dropRole(session.toConnectorSession(connectorId), role); + metadata.dropRole(session.toConnectorSession(connectorId), role); + } } @Override public Set listRoles(Session session, String catalog) { - Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); - if (!catalogMetadata.isPresent()) { - return ImmutableSet.of(); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_ROLES.getName(), skipSpan)) { + Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); + if (!catalogMetadata.isPresent()) { + return ImmutableSet.of(); + } + ConnectorId connectorId = catalogMetadata.get().getConnectorId(); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); + return metadata.listRoles(connectorSession).stream() + .map(role -> role.toLowerCase(ENGLISH)) + .collect(toImmutableSet()); } - ConnectorId connectorId = catalogMetadata.get().getConnectorId(); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); - return metadata.listRoles(connectorSession).stream() - .map(role -> role.toLowerCase(ENGLISH)) - .collect(toImmutableSet()); } @Override public Set listRoleGrants(Session session, String catalog, PrestoPrincipal principal) { - Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); - if (!catalogMetadata.isPresent()) { - return ImmutableSet.of(); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_ROLE_GRANTS.getName(), skipSpan)) { + Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); + if (!catalogMetadata.isPresent()) { + return ImmutableSet.of(); + } + ConnectorId connectorId = catalogMetadata.get().getConnectorId(); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); + return metadata.listRoleGrants(connectorSession, principal); } - ConnectorId connectorId = catalogMetadata.get().getConnectorId(); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); - return metadata.listRoleGrants(connectorSession, principal); } @Override public void grantRoles(Session session, Set roles, Set grantees, boolean withAdminOption, Optional grantor, String catalog) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GRANT_ROLES.getName(), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.grantRoles(session.toConnectorSession(connectorId), roles, grantees, withAdminOption, grantor); + metadata.grantRoles(session.toConnectorSession(connectorId), roles, grantees, withAdminOption, grantor); + } } @Override public void revokeRoles(Session session, Set roles, Set grantees, boolean adminOptionFor, Optional grantor, String catalog) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.REVOKE_ROLES.getName(), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.revokeRoles(session.toConnectorSession(connectorId), roles, grantees, adminOptionFor, grantor); + metadata.revokeRoles(session.toConnectorSession(connectorId), roles, grantees, adminOptionFor, grantor); + } } @Override public Set listApplicableRoles(Session session, PrestoPrincipal principal, String catalog) { - Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); - if (!catalogMetadata.isPresent()) { - return ImmutableSet.of(); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_APPLICABLE_ROLES.getName(), skipSpan)) { + Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); + if (!catalogMetadata.isPresent()) { + return ImmutableSet.of(); + } + ConnectorId connectorId = catalogMetadata.get().getConnectorId(); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); + return ImmutableSet.copyOf(metadata.listApplicableRoles(connectorSession, principal)); } - ConnectorId connectorId = catalogMetadata.get().getConnectorId(); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); - return ImmutableSet.copyOf(metadata.listApplicableRoles(connectorSession, principal)); } @Override public Set listEnabledRoles(Session session, String catalog) { - Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); - if (!catalogMetadata.isPresent()) { - return ImmutableSet.of(); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_ENABLED_ROLES.getName(), skipSpan)) { + Optional catalogMetadata = getOptionalCatalogMetadata(session, transactionManager, catalog); + if (!catalogMetadata.isPresent()) { + return ImmutableSet.of(); + } + ConnectorId connectorId = catalogMetadata.get().getConnectorId(); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); + ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); + return ImmutableSet.copyOf(metadata.listEnabledRoles(connectorSession)); } - ConnectorId connectorId = catalogMetadata.get().getConnectorId(); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); - ConnectorMetadata metadata = catalogMetadata.get().getMetadataFor(connectorId); - return ImmutableSet.copyOf(metadata.listEnabledRoles(connectorSession)); } @Override public void grantTablePrivileges(Session session, QualifiedObjectName tableName, Set privileges, PrestoPrincipal grantee, boolean grantOption) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, tableName.getCatalogName()); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GRANT_TABLE_PRIVILEGES.getName(), ImmutableMap.of("CATALOG", tableName.getCatalogName(), "SCHEMA", tableName.getSchemaName(), "TABLE", tableName.getObjectName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, tableName.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.grantTablePrivileges(session.toConnectorSession(connectorId), toSchemaTableName(tableName), privileges, grantee, grantOption); + metadata.grantTablePrivileges(session.toConnectorSession(connectorId), toSchemaTableName(tableName), privileges, grantee, grantOption); + } } @Override public void revokeTablePrivileges(Session session, QualifiedObjectName tableName, Set privileges, PrestoPrincipal grantee, boolean grantOption) { - CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, tableName.getCatalogName()); - ConnectorId connectorId = catalogMetadata.getConnectorId(); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.REVOKE_TABLE_PRIVILEGES.getName(), ImmutableMap.of("CATALOG", tableName.getCatalogName(), "SCHEMA", tableName.getSchemaName(), "TABLE", tableName.getObjectName()), skipSpan)) { + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, tableName.getCatalogName()); + ConnectorId connectorId = catalogMetadata.getConnectorId(); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); - metadata.revokeTablePrivileges(session.toConnectorSession(connectorId), toSchemaTableName(tableName), privileges, grantee, grantOption); + metadata.revokeTablePrivileges(session.toConnectorSession(connectorId), toSchemaTableName(tableName), privileges, grantee, grantOption); + } } @Override public List listTablePrivileges(Session session, QualifiedTablePrefix prefix) { - requireNonNull(prefix, "prefix is null"); - SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix(); + try (BaseSpan ignored = scopedSpan(TracingEnum.LIST_TABLE_PRIVILEGES.getName(), ImmutableMap.of("CATALOG", prefix.getCatalogName(), "SCHEMA", prefix.getSchemaName().orElse(""), "TABLE", prefix.getTableName().orElse("")), skipSpan)) { + requireNonNull(prefix, "prefix is null"); + SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix(); - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, prefix.getCatalogName()); - ImmutableSet.Builder grantInfos = ImmutableSet.builder(); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); - ConnectorSession connectorSession = session.toConnectorSession(catalogMetadata.getConnectorId()); - for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - grantInfos.addAll(metadata.listTablePrivileges(connectorSession, tablePrefix)); + ImmutableSet.Builder grantInfos = ImmutableSet.builder(); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorSession connectorSession = session.toConnectorSession(catalogMetadata.getConnectorId()); + for (ConnectorId connectorId : catalogMetadata.listConnectorIds()) { + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + grantInfos.addAll(metadata.listTablePrivileges(connectorSession, tablePrefix)); + } } + return ImmutableList.copyOf(grantInfos.build()); } - return ImmutableList.copyOf(grantInfos.build()); } @Override public ListenableFuture commitPageSinkAsync(Session session, OutputTableHandle tableHandle, Collection fragments) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.COMMIT_PAGE_SINK_ASYNC.getName(), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); - return toListenableFuture(metadata.commitPageSinkAsync(connectorSession, tableHandle.getConnectorHandle(), fragments)); + return toListenableFuture(metadata.commitPageSinkAsync(connectorSession, tableHandle.getConnectorHandle(), fragments)); + } } @Override public ListenableFuture commitPageSinkAsync(Session session, InsertTableHandle tableHandle, Collection fragments) { - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadata(); - ConnectorSession connectorSession = session.toConnectorSession(connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.COMMIT_PAGE_SINK_ASYNC.getName(), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadata(); + ConnectorSession connectorSession = session.toConnectorSession(connectorId); - return toListenableFuture(metadata.commitPageSinkAsync(connectorSession, tableHandle.getConnectorHandle(), fragments)); + return toListenableFuture(metadata.commitPageSinkAsync(connectorSession, tableHandle.getConnectorHandle(), fragments)); + } } @Override public MetadataUpdates getMetadataUpdateResults(Session session, QueryManager queryManager, MetadataUpdates metadataUpdateRequests, QueryId queryId) { - ConnectorId connectorId = metadataUpdateRequests.getConnectorId(); - ConnectorMetadata metadata = getCatalogMetadata(session, connectorId).getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_METADATA_UPDATE_RESULTS.getName(), skipSpan)) { + ConnectorId connectorId = metadataUpdateRequests.getConnectorId(); + ConnectorMetadata metadata = getCatalogMetadata(session, connectorId).getMetadata(); + + if (queryManager != null && !queriesWithRegisteredCallbacks.contains(queryId)) { + // This is the first time we are getting requests for queryId. + // Register a callback, so the we do the cleanup when query fails/finishes. + queryManager.addStateChangeListener(queryId, state -> { + if (state.isDone()) { + metadata.doMetadataUpdateCleanup(queryId); + queriesWithRegisteredCallbacks.remove(queryId); + } + }); + queriesWithRegisteredCallbacks.add(queryId); + } - if (queryManager != null && !queriesWithRegisteredCallbacks.contains(queryId)) { - // This is the first time we are getting requests for queryId. - // Register a callback, so the we do the cleanup when query fails/finishes. - queryManager.addStateChangeListener(queryId, state -> { - if (state.isDone()) { - metadata.doMetadataUpdateCleanup(queryId); - queriesWithRegisteredCallbacks.remove(queryId); - } - }); - queriesWithRegisteredCallbacks.add(queryId); + List metadataResults = metadata.getMetadataUpdateResults(metadataUpdateRequests.getMetadataUpdates(), queryId); + return new MetadataUpdates(connectorId, metadataResults); } - - List metadataResults = metadata.getMetadataUpdateResults(metadataUpdateRequests.getMetadataUpdates(), queryId); - return new MetadataUpdates(connectorId, metadataResults); } @Override @@ -1333,169 +1494,209 @@ public AnalyzePropertyManager getAnalyzePropertyManager() @Override public MetadataResolver getMetadataResolver(Session session) { - return new MetadataResolver() - { - @Override - public boolean catalogExists(String catalogName) - { - return getOptionalCatalogMetadata(session, transactionManager, catalogName).isPresent(); - } - - @Override - public boolean schemaExists(CatalogSchemaName schema) + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_METADATA_RESOLVER.getName(), skipSpan)) { + return new MetadataResolver() { - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, schema.getCatalogName()); - if (!catalog.isPresent()) { - return false; + @Override + public boolean catalogExists(String catalogName) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.CATALOG_EXISTS.getName(), ImmutableMap.of("CATALOG", catalogName), skipSpan)) { + return getOptionalCatalogMetadata(session, transactionManager, catalogName).isPresent(); + } } - CatalogMetadata catalogMetadata = catalog.get(); - ConnectorSession connectorSession = session.toConnectorSession(catalogMetadata.getConnectorId()); - return catalogMetadata.listConnectorIds().stream() - .map(catalogMetadata::getMetadataFor) - .anyMatch(metadata -> metadata.schemaExists(connectorSession, schema.getSchemaName())); - } - @Override - public boolean tableExists(QualifiedObjectName tableName) - { - return getOptionalTableHandle(session, transactionManager, tableName, Optional.empty()).isPresent(); - } + @Override + public boolean schemaExists(CatalogSchemaName schema) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.SCHEMA_EXISTS.getName(), ImmutableMap.of("CATALOG", schema.getCatalogName(), "SCHEMA", schema.getSchemaName()), skipSpan)) { + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, schema.getCatalogName()); + if (!catalog.isPresent()) { + return false; + } + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorSession connectorSession = session.toConnectorSession(catalogMetadata.getConnectorId()); + return catalogMetadata.listConnectorIds().stream() + .map(catalogMetadata::getMetadataFor) + .anyMatch(metadata -> metadata.schemaExists(connectorSession, schema.getSchemaName())); + } + } - @Override - public Optional getTableHandle(QualifiedObjectName tableName) - { - return getOptionalTableHandle(session, transactionManager, tableName, Optional.empty()); - } + @Override + public boolean tableExists(QualifiedObjectName tableName) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.TABLE_EXISTS.getName(), ImmutableMap.of("CATALOG", tableName.getCatalogName(), "SCHEMA", tableName.getSchemaName(), "TABLE", tableName.getObjectName()), skipSpan)) { + return getOptionalTableHandle(session, transactionManager, tableName, Optional.empty()).isPresent(); + } + } - @Override - public List getColumns(TableHandle tableHandle) - { - return getTableMetadata(session, tableHandle).getColumns(); - } + @Override + public Optional getTableHandle(QualifiedObjectName tableName) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_TABLE_HANDLE.getName(), ImmutableMap.of("CATALOG", tableName.getCatalogName(), "SCHEMA", tableName.getSchemaName(), "TABLE", tableName.getObjectName()), skipSpan)) { + return getOptionalTableHandle(session, transactionManager, tableName, Optional.empty()); + } + } - @Override - public Map getColumnHandles(TableHandle tableHandle) - { - return MetadataManager.this.getColumnHandles(session, tableHandle); - } + @Override + public List getColumns(TableHandle tableHandle) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_COLUMNS.getName(), skipSpan)) { + return getTableMetadata(session, tableHandle).getColumns(); + } + } - @Override - public Optional getView(QualifiedObjectName viewName) - { - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, viewName.getCatalogName()); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); - ConnectorId connectorId = catalogMetadata.getConnectorId(session, viewName); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + @Override + public Map getColumnHandles(TableHandle tableHandle) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_COLUMN_HANDLES.getName(), skipSpan)) { + return MetadataManager.this.getColumnHandles(session, tableHandle); + } + } - Map views = metadata.getViews( - session.toConnectorSession(connectorId), - toSchemaTableName(viewName).toSchemaTablePrefix()); - ConnectorViewDefinition view = views.get(toSchemaTableName(viewName)); - if (view != null) { - ViewDefinition definition = deserializeView(view.getViewData()); - if (view.getOwner().isPresent() && !definition.isRunAsInvoker()) { - definition = definition.withOwner(view.getOwner().get()); + @Override + public Optional getView(QualifiedObjectName viewName) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_VIEW.getName(), ImmutableMap.of("CATALOG", viewName.getCatalogName(), "SCHEMA", viewName.getSchemaName(), "TABLE", viewName.getObjectName()), skipSpan)) { + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, viewName.getCatalogName()); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorId connectorId = catalogMetadata.getConnectorId(session, viewName); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + + Map views = metadata.getViews( + session.toConnectorSession(connectorId), + toSchemaTableName(viewName).toSchemaTablePrefix()); + ConnectorViewDefinition view = views.get(toSchemaTableName(viewName)); + if (view != null) { + ViewDefinition definition = deserializeView(view.getViewData()); + if (view.getOwner().isPresent() && !definition.isRunAsInvoker()) { + definition = definition.withOwner(view.getOwner().get()); + } + return Optional.of(definition); + } } - return Optional.of(definition); + return Optional.empty(); } } - return Optional.empty(); - } - - @Override - public Optional getMaterializedView(QualifiedObjectName viewName) - { - Optional catalog = getOptionalCatalogMetadata(session, transactionManager, viewName.getCatalogName()); - if (catalog.isPresent()) { - CatalogMetadata catalogMetadata = catalog.get(); - ConnectorId connectorId = catalogMetadata.getConnectorId(session, viewName); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - return metadata.getMaterializedView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + @Override + public Optional getMaterializedView(QualifiedObjectName viewName) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_MATERIALIZED_VIEW.getName(), ImmutableMap.of("CATALOG", viewName.getCatalogName(), "SCHEMA", viewName.getSchemaName(), "TABLE", viewName.getObjectName()), skipSpan)) { + Optional catalog = getOptionalCatalogMetadata(session, transactionManager, viewName.getCatalogName()); + if (catalog.isPresent()) { + CatalogMetadata catalogMetadata = catalog.get(); + ConnectorId connectorId = catalogMetadata.getConnectorId(session, viewName); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + + return metadata.getMaterializedView(session.toConnectorSession(connectorId), toSchemaTableName(viewName)); + } + return Optional.empty(); + } } - return Optional.empty(); - } - @Override - public MaterializedViewStatus getMaterializedViewStatus(QualifiedObjectName materializedViewName, TupleDomain baseQueryDomain) - { - return MetadataManager.this.getMaterializedViewStatus(session, materializedViewName, baseQueryDomain); - } - }; + @Override + public MaterializedViewStatus getMaterializedViewStatus(QualifiedObjectName materializedViewName, TupleDomain baseQueryDomain) + { + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_MATERIALIZED_VIEW_STATUS.getName(), ImmutableMap.of("CATALOG", materializedViewName.getCatalogName(), "SCHEMA", materializedViewName.getSchemaName(), "TABLE", materializedViewName.getObjectName()), skipSpan)) { + return MetadataManager.this.getMaterializedViewStatus(session, materializedViewName, baseQueryDomain); + } + } + }; + } } @Override public Set getConnectorCapabilities(Session session, ConnectorId connectorId) { - return getCatalogMetadata(session, connectorId).getConnectorCapabilities(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_CONNECTOR_CAPABILITIES.getName(), skipSpan)) { + return getCatalogMetadata(session, connectorId).getConnectorCapabilities(); + } } @Override public TableLayoutFilterCoverage getTableLayoutFilterCoverage(Session session, TableHandle tableHandle, Set relevantPartitionColumns) { - requireNonNull(tableHandle, "tableHandle cannot be null"); - requireNonNull(relevantPartitionColumns, "relevantPartitionKeys cannot be null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_TABLE_LAYOUT_FILTER_COVERAGE.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + requireNonNull(tableHandle, "tableHandle cannot be null"); + requireNonNull(relevantPartitionColumns, "relevantPartitionKeys cannot be null"); - if (!tableHandle.getLayout().isPresent()) { - return NOT_APPLICABLE; - } + if (!tableHandle.getLayout().isPresent()) { + return NOT_APPLICABLE; + } - ConnectorId connectorId = tableHandle.getConnectorId(); - CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); - ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); - return metadata.getTableLayoutFilterCoverage(tableHandle.getLayout().get(), relevantPartitionColumns); + ConnectorId connectorId = tableHandle.getConnectorId(); + CatalogMetadata catalogMetadata = getCatalogMetadata(session, connectorId); + ConnectorMetadata metadata = catalogMetadata.getMetadataFor(connectorId); + return metadata.getTableLayoutFilterCoverage(tableHandle.getLayout().get(), relevantPartitionColumns); + } } @Override public void dropConstraint(Session session, TableHandle tableHandle, Optional constraintName, Optional columnName) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); - metadata.dropConstraint(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), constraintName, columnName); + try (BaseSpan ignored = scopedSpan(TracingEnum.DROP_CONSTRAINT.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); + metadata.dropConstraint(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), constraintName, columnName); + } } @Override public void addConstraint(Session session, TableHandle tableHandle, TableConstraint tableConstraint) { - ConnectorId connectorId = tableHandle.getConnectorId(); - ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); - metadata.addConstraint(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableConstraint); + try (BaseSpan ignored = scopedSpan(TracingEnum.ADD_CONSTRAINT.getName(), ImmutableMap.of("HANDLE", tableHandle.getConnectorHandle().toString()), skipSpan)) { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); + metadata.addConstraint(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tableConstraint); + } } private ViewDefinition deserializeView(String data) { - try { - return viewCodec.fromJson(data); - } - catch (IllegalArgumentException e) { - throw new PrestoException(INVALID_VIEW, "Invalid view JSON: " + data, e); + try (BaseSpan ignored = scopedSpan(TracingEnum.DESERIALIZE_VIEW.getName(), skipSpan)) { + try { + return viewCodec.fromJson(data); + } + catch (IllegalArgumentException e) { + throw new PrestoException(INVALID_VIEW, "Invalid view JSON: " + data, e); + } } } private CatalogMetadata getCatalogMetadata(Session session, ConnectorId connectorId) { - return transactionManager.getCatalogMetadata(session.getRequiredTransactionId(), connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_CATALOG_METADATA.getName(), skipSpan)) { + return transactionManager.getCatalogMetadata(session.getRequiredTransactionId(), connectorId); + } } private CatalogMetadata getCatalogMetadataForWrite(Session session, String catalogName) { - return transactionManager.getCatalogMetadataForWrite(session.getRequiredTransactionId(), catalogName); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_CATALOG_METADATA_FOR_WRITE.getName(), ImmutableMap.of("CATALOG", catalogName), skipSpan)) { + return transactionManager.getCatalogMetadataForWrite(session.getRequiredTransactionId(), catalogName); + } } private CatalogMetadata getCatalogMetadataForWrite(Session session, ConnectorId connectorId) { - return transactionManager.getCatalogMetadataForWrite(session.getRequiredTransactionId(), connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_CATALOG_METADATA_FOR_WRITE.getName(), skipSpan)) { + return transactionManager.getCatalogMetadataForWrite(session.getRequiredTransactionId(), connectorId); + } } private ConnectorMetadata getMetadata(Session session, ConnectorId connectorId) { - return getCatalogMetadata(session, connectorId).getMetadataFor(connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_METADATA.getName(), skipSpan)) { + return getCatalogMetadata(session, connectorId).getMetadataFor(connectorId); + } } private ConnectorMetadata getMetadataForWrite(Session session, ConnectorId connectorId) { - return getCatalogMetadataForWrite(session, connectorId).getMetadata(); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_METADATA_FOR_WRITE.getName(), skipSpan)) { + return getCatalogMetadataForWrite(session, connectorId).getMetadata(); + } } private static JsonCodec createTestingViewCodec(FunctionAndTypeManager functionAndTypeManager) @@ -1518,17 +1719,21 @@ private boolean canResolveOperator(OperatorType operatorType, List> getCatalogsByQueryId() { - return ImmutableMap.copyOf(catalogsByQueryId); + try (BaseSpan ignored = scopedSpan(TracingEnum.GET_CATALOGS_BY_QUERY_ID.getName(), skipSpan)) { + return ImmutableMap.copyOf(catalogsByQueryId); + } } public static Function convertFromSchemaTableName(String catalogName) diff --git a/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java b/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java index 468d7d0ed2f3b..87c969a0d52d7 100644 --- a/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java +++ b/presto-main/src/main/java/com/facebook/presto/security/AccessControlManager.java @@ -18,6 +18,8 @@ import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.Subfield; +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.common.transaction.TransactionId; import com.facebook.presto.spi.CatalogSchemaTableName; import com.facebook.presto.spi.ConnectorId; @@ -33,6 +35,7 @@ import com.facebook.presto.spi.security.Privilege; import com.facebook.presto.spi.security.SystemAccessControl; import com.facebook.presto.spi.security.SystemAccessControlFactory; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.transaction.TransactionManager; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -56,6 +59,7 @@ import static com.facebook.presto.metadata.MetadataUtil.toSchemaTableName; import static com.facebook.presto.spi.StandardErrorCode.SERVER_STARTING_UP; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; import static com.facebook.presto.util.PropertiesUtil.loadProperties; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -83,6 +87,8 @@ public class AccessControlManager private final CounterStat authorizationSuccess = new CounterStat(); private final CounterStat authorizationFail = new CounterStat(); + private Boolean skipSpan = TelemetryConfig.getSpanSampling(); + @Inject public AccessControlManager(TransactionManager transactionManager) { @@ -94,246 +100,284 @@ public AccessControlManager(TransactionManager transactionManager) public void addSystemAccessControlFactory(SystemAccessControlFactory accessControlFactory) { - requireNonNull(accessControlFactory, "accessControlFactory is null"); + try (BaseSpan ignored = scopedSpan(TracingEnum.ADD_SYSTEM_ACCESS_CONTROL_FACTORY.getName(), skipSpan)) { + requireNonNull(accessControlFactory, "accessControlFactory is null"); - if (systemAccessControlFactories.putIfAbsent(accessControlFactory.getName(), accessControlFactory) != null) { - throw new IllegalArgumentException(format("Access control '%s' is already registered", accessControlFactory.getName())); + if (systemAccessControlFactories.putIfAbsent(accessControlFactory.getName(), accessControlFactory) != null) { + throw new IllegalArgumentException(format("Access control '%s' is already registered", accessControlFactory.getName())); + } } } public void addCatalogAccessControl(ConnectorId connectorId, ConnectorAccessControl accessControl) { - requireNonNull(connectorId, "connectorId is null"); - requireNonNull(accessControl, "accessControl is null"); - checkState(connectorAccessControl.putIfAbsent(connectorId, new CatalogAccessControlEntry(connectorId, accessControl)) == null, - "Access control for connector '%s' is already registered", connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.ADD_CATALOG_ACCESS_CONTROL.getName(), skipSpan)) { + requireNonNull(connectorId, "connectorId is null"); + requireNonNull(accessControl, "accessControl is null"); + checkState(connectorAccessControl.putIfAbsent(connectorId, new CatalogAccessControlEntry(connectorId, accessControl)) == null, + "Access control for connector '%s' is already registered", connectorId); + } } public void removeCatalogAccessControl(ConnectorId connectorId) { - connectorAccessControl.remove(connectorId); + try (BaseSpan ignored = scopedSpan(TracingEnum.REMOVE_CATALOG_ACCESS_CONTROL.getName(), skipSpan)) { + connectorAccessControl.remove(connectorId); + } } public void loadSystemAccessControl() throws Exception { - if (ACCESS_CONTROL_CONFIGURATION.exists()) { - Map properties = loadProperties(ACCESS_CONTROL_CONFIGURATION); - checkArgument(!isNullOrEmpty(properties.get(ACCESS_CONTROL_PROPERTY_NAME)), - "Access control configuration %s does not contain %s", - ACCESS_CONTROL_CONFIGURATION.getAbsoluteFile(), - ACCESS_CONTROL_PROPERTY_NAME); + try (BaseSpan ignored = scopedSpan(TracingEnum.LOAD_CATALOG_ACCESS_CONTROL.getName(), skipSpan)) { + if (ACCESS_CONTROL_CONFIGURATION.exists()) { + Map properties = loadProperties(ACCESS_CONTROL_CONFIGURATION); + checkArgument(!isNullOrEmpty(properties.get(ACCESS_CONTROL_PROPERTY_NAME)), + "Access control configuration %s does not contain %s", + ACCESS_CONTROL_CONFIGURATION.getAbsoluteFile(), + ACCESS_CONTROL_PROPERTY_NAME); - loadSystemAccessControl(properties); - } - else { - setSystemAccessControl(AllowAllSystemAccessControl.NAME, ImmutableMap.of()); + loadSystemAccessControl(properties); + } + else { + setSystemAccessControl(AllowAllSystemAccessControl.NAME, ImmutableMap.of()); + } } } public void loadSystemAccessControl(Map properties) { - properties = new HashMap<>(properties); - String accessControlName = properties.remove(ACCESS_CONTROL_PROPERTY_NAME); - checkArgument(!isNullOrEmpty(accessControlName), "%s property must be present", ACCESS_CONTROL_PROPERTY_NAME); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.LOAD_SYSTEM_ACCESS_CONTROL.getName(), skipSpan)) { + properties = new HashMap<>(properties); + String accessControlName = properties.remove(ACCESS_CONTROL_PROPERTY_NAME); + checkArgument(!isNullOrEmpty(accessControlName), "%s property must be present", ACCESS_CONTROL_PROPERTY_NAME); - setSystemAccessControl(accessControlName, properties); + setSystemAccessControl(accessControlName, properties); + } } @VisibleForTesting protected void setSystemAccessControl(String name, Map properties) { - requireNonNull(name, "name is null"); - requireNonNull(properties, "properties is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.SET_SYSTEM_ACCESS_CONTROL.getName(), skipSpan)) { + requireNonNull(name, "name is null"); + requireNonNull(properties, "properties is null"); - checkState(systemAccessControlLoading.compareAndSet(false, true), "System access control already initialized"); + checkState(systemAccessControlLoading.compareAndSet(false, true), "System access control already initialized"); - log.info("-- Loading system access control --"); + log.info("-- Loading system access control --"); - SystemAccessControlFactory systemAccessControlFactory = systemAccessControlFactories.get(name); - checkState(systemAccessControlFactory != null, "Access control %s is not registered", name); + SystemAccessControlFactory systemAccessControlFactory = systemAccessControlFactories.get(name); + checkState(systemAccessControlFactory != null, "Access control %s is not registered", name); - SystemAccessControl systemAccessControl = systemAccessControlFactory.create(ImmutableMap.copyOf(properties)); - this.systemAccessControl.set(systemAccessControl); + SystemAccessControl systemAccessControl = systemAccessControlFactory.create(ImmutableMap.copyOf(properties)); + this.systemAccessControl.set(systemAccessControl); - log.info("-- Loaded system access control %s --", name); + log.info("-- Loaded system access control %s --", name); + } } @Override public void checkCanSetUser(Identity identity, AccessControlContext context, Optional principal, String userName) { - requireNonNull(principal, "principal is null"); - requireNonNull(userName, "userName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SET_USER.getName(), skipSpan)) { + requireNonNull(principal, "principal is null"); + requireNonNull(userName, "userName is null"); - authenticationCheck(() -> systemAccessControl.get().checkCanSetUser(identity, context, principal, userName)); + authenticationCheck(() -> systemAccessControl.get().checkCanSetUser(identity, context, principal, userName)); + } } @Override public AuthorizedIdentity selectAuthorizedIdentity(Identity identity, AccessControlContext context, String userName, List certificates) { - requireNonNull(userName, "userName is null"); - requireNonNull(certificates, "certificates is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.SELECT_AUTHORIZED_IDENTITY.getName(), skipSpan)) { + requireNonNull(userName, "userName is null"); + requireNonNull(certificates, "certificates is null"); - return systemAccessControl.get().selectAuthorizedIdentity(identity, context, userName, certificates); + return systemAccessControl.get().selectAuthorizedIdentity(identity, context, userName, certificates); + } } @Override public void checkQueryIntegrity(Identity identity, AccessControlContext context, String query) { - requireNonNull(identity, "identity is null"); - requireNonNull(query, "query is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_QUERY_INTEGRITY.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(query, "query is null"); - authenticationCheck(() -> systemAccessControl.get().checkQueryIntegrity(identity, context, query)); + authenticationCheck(() -> systemAccessControl.get().checkQueryIntegrity(identity, context, query)); + } } @Override public Set filterCatalogs(Identity identity, AccessControlContext context, Set catalogs) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogs, "catalogs is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.FILTER_CATALOGS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogs, "catalogs is null"); - return systemAccessControl.get().filterCatalogs(identity, context, catalogs); + return systemAccessControl.get().filterCatalogs(identity, context, catalogs); + } } @Override public void checkCanAccessCatalog(Identity identity, AccessControlContext context, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalog is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_ACCESS_CATALOG.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalog is null"); - authenticationCheck(() -> systemAccessControl.get().checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> systemAccessControl.get().checkCanAccessCatalog(identity, context, catalogName)); + } } @Override public void checkCanCreateSchema(TransactionId transactionId, Identity identity, AccessControlContext context, CatalogSchemaName schemaName) { - requireNonNull(identity, "identity is null"); - requireNonNull(schemaName, "schemaName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_CREATE_SCHEMA.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(schemaName, "schemaName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateSchema(identity, context, schemaName)); + authorizationCheck(() -> systemAccessControl.get().checkCanCreateSchema(identity, context, schemaName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanCreateSchema(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(schemaName.getCatalogName()), context, schemaName.getSchemaName())); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanCreateSchema(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(schemaName.getCatalogName()), context, schemaName.getSchemaName())); + } } } @Override public void checkCanDropSchema(TransactionId transactionId, Identity identity, AccessControlContext context, CatalogSchemaName schemaName) { - requireNonNull(identity, "identity is null"); - requireNonNull(schemaName, "schemaName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_DROP_SCHEMA.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(schemaName, "schemaName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropSchema(identity, context, schemaName)); + authorizationCheck(() -> systemAccessControl.get().checkCanDropSchema(identity, context, schemaName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanDropSchema(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(schemaName.getCatalogName()), context, schemaName.getSchemaName())); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanDropSchema(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(schemaName.getCatalogName()), context, schemaName.getSchemaName())); + } } } @Override public void checkCanRenameSchema(TransactionId transactionId, Identity identity, AccessControlContext context, CatalogSchemaName schemaName, String newSchemaName) { - requireNonNull(identity, "identity is null"); - requireNonNull(schemaName, "schemaName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_RENAME_SCHEMA.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(schemaName, "schemaName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRenameSchema(identity, context, schemaName, newSchemaName)); + authorizationCheck(() -> systemAccessControl.get().checkCanRenameSchema(identity, context, schemaName, newSchemaName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanRenameSchema(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(schemaName.getCatalogName()), context, schemaName.getSchemaName(), newSchemaName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanRenameSchema(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(schemaName.getCatalogName()), context, schemaName.getSchemaName(), newSchemaName)); + } } } @Override public void checkCanShowSchemas(TransactionId transactionId, Identity identity, AccessControlContext context, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SHOW_SCHEMAS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - authorizationCheck(() -> systemAccessControl.get().checkCanShowSchemas(identity, context, catalogName)); + authorizationCheck(() -> systemAccessControl.get().checkCanShowSchemas(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanShowSchemas(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanShowSchemas(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context)); + } } } @Override public Set filterSchemas(TransactionId transactionId, Identity identity, AccessControlContext context, String catalogName, Set schemaNames) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalogName is null"); - requireNonNull(schemaNames, "schemaNames is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.FILTER_SCHEMAS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalogName is null"); + requireNonNull(schemaNames, "schemaNames is null"); - if (filterCatalogs(identity, context, ImmutableSet.of(catalogName)).isEmpty()) { - return ImmutableSet.of(); - } + if (filterCatalogs(identity, context, ImmutableSet.of(catalogName)).isEmpty()) { + return ImmutableSet.of(); + } - schemaNames = systemAccessControl.get().filterSchemas(identity, context, catalogName, schemaNames); + schemaNames = systemAccessControl.get().filterSchemas(identity, context, catalogName, schemaNames); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - schemaNames = entry.getAccessControl().filterSchemas(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, schemaNames); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + schemaNames = entry.getAccessControl().filterSchemas(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, schemaNames); + } + return schemaNames; } - return schemaNames; } @Override public void checkCanCreateTable(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_CREATE_TABLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanCreateTable(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanCreateTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanCreateTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanDropTable(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_DROP_TABLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanDropTable(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanDropTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanDropTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanRenameTable(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName, QualifiedObjectName newTableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); - requireNonNull(newTableName, "newTableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_RENAME_TABLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); + requireNonNull(newTableName, "newTableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRenameTable(identity, context, toCatalogSchemaTableName(tableName), toCatalogSchemaTableName(newTableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanRenameTable(identity, context, toCatalogSchemaTableName(tableName), toCatalogSchemaTableName(newTableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanRenameTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), toSchemaTableName(newTableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanRenameTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), toSchemaTableName(newTableName))); + } } } @@ -353,164 +397,184 @@ public void checkCanSetTableProperties(TransactionId transactionId, Identity ide @Override public void checkCanShowTablesMetadata(TransactionId transactionId, Identity identity, AccessControlContext context, CatalogSchemaName schema) { - requireNonNull(identity, "identity is null"); - requireNonNull(schema, "schema is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SHOW_TABLES_METADATA.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(schema, "schema is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, schema.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, schema.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanShowTablesMetadata(identity, context, schema)); + authorizationCheck(() -> systemAccessControl.get().checkCanShowTablesMetadata(identity, context, schema)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schema.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanShowTablesMetadata(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(), context, schema.getSchemaName())); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schema.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanShowTablesMetadata(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(), context, schema.getSchemaName())); + } } } @Override public Set filterTables(TransactionId transactionId, Identity identity, AccessControlContext context, String catalogName, Set tableNames) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalogName is null"); - requireNonNull(tableNames, "tableNames is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.FILTER_TABLES.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalogName is null"); + requireNonNull(tableNames, "tableNames is null"); - if (filterCatalogs(identity, context, ImmutableSet.of(catalogName)).isEmpty()) { - return ImmutableSet.of(); - } + if (filterCatalogs(identity, context, ImmutableSet.of(catalogName)).isEmpty()) { + return ImmutableSet.of(); + } - tableNames = systemAccessControl.get().filterTables(identity, context, catalogName, tableNames); + tableNames = systemAccessControl.get().filterTables(identity, context, catalogName, tableNames); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - tableNames = entry.getAccessControl().filterTables(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, tableNames); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + tableNames = entry.getAccessControl().filterTables(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, tableNames); + } + return tableNames; } - return tableNames; } @Override public void checkCanAddColumns(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_ADD_COLUMNS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanAddColumn(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanAddColumn(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanAddColumn(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanAddColumn(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanDropColumn(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_DROP_COLUMN.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropColumn(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanDropColumn(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanDropColumn(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanDropColumn(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanRenameColumn(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_RENAME_COLUMN.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRenameColumn(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanRenameColumn(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanRenameColumn(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanRenameColumn(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanInsertIntoTable(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_INSERT_INTO_TABLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanInsertIntoTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanInsertIntoTable(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanInsertIntoTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanInsertIntoTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanDeleteFromTable(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_DELETE_FROM_TABLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDeleteFromTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanDeleteFromTable(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanDeleteFromTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanDeleteFromTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanTruncateTable(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_TRUNCATE_TABLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanTruncateTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanTruncateTable(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanTruncateTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanTruncateTable(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanUpdateTableColumns(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName, Set updatedColumnNames) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_UPDATE_TABLE_COLUMNS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanUpdateTableColumns(identity, context, toCatalogSchemaTableName(tableName), updatedColumnNames)); + authorizationCheck(() -> systemAccessControl.get().checkCanUpdateTableColumns(identity, context, toCatalogSchemaTableName(tableName), updatedColumnNames)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanUpdateTableColumns(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), updatedColumnNames)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanUpdateTableColumns(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), updatedColumnNames)); + } } } @Override public void checkCanCreateView(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName viewName) { - requireNonNull(identity, "identity is null"); - requireNonNull(viewName, "viewName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_CREATE_VIEW.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(viewName, "viewName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, viewName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, viewName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateView(identity, context, toCatalogSchemaTableName(viewName))); + authorizationCheck(() -> systemAccessControl.get().checkCanCreateView(identity, context, toCatalogSchemaTableName(viewName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, viewName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanCreateView(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(viewName.getCatalogName()), context, toSchemaTableName(viewName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, viewName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanCreateView(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(viewName.getCatalogName()), context, toSchemaTableName(viewName))); + } } } @@ -534,275 +598,311 @@ public void checkCanRenameView(TransactionId transactionId, Identity identity, A @Override public void checkCanDropView(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName viewName) { - requireNonNull(identity, "identity is null"); - requireNonNull(viewName, "viewName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_DROP_VIEW.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(viewName, "viewName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, viewName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, viewName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropView(identity, context, toCatalogSchemaTableName(viewName))); + authorizationCheck(() -> systemAccessControl.get().checkCanDropView(identity, context, toCatalogSchemaTableName(viewName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, viewName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanDropView(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(viewName.getCatalogName()), context, toSchemaTableName(viewName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, viewName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanDropView(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(viewName.getCatalogName()), context, toSchemaTableName(viewName))); + } } } @Override public void checkCanCreateViewWithSelectFromColumns(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName, Set columnNames) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_CREATE_VIEW_WITH_SELECT_FROM_COLUMNS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateViewWithSelectFromColumns(identity, context, toCatalogSchemaTableName(tableName), columnNames)); + authorizationCheck(() -> systemAccessControl.get().checkCanCreateViewWithSelectFromColumns(identity, context, toCatalogSchemaTableName(tableName), columnNames)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanCreateViewWithSelectFromColumns(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), columnNames)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanCreateViewWithSelectFromColumns(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), columnNames)); + } } } @Override public void checkCanGrantTablePrivilege(TransactionId transactionId, Identity identity, AccessControlContext context, Privilege privilege, QualifiedObjectName tableName, PrestoPrincipal grantee, boolean withGrantOption) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); - requireNonNull(privilege, "privilege is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_GRANT_TABLE_PRIVILEGE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); + requireNonNull(privilege, "privilege is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanGrantTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), grantee, withGrantOption)); + authorizationCheck(() -> systemAccessControl.get().checkCanGrantTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), grantee, withGrantOption)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanGrantTablePrivilege(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, privilege, toSchemaTableName(tableName), grantee, withGrantOption)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanGrantTablePrivilege(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, privilege, toSchemaTableName(tableName), grantee, withGrantOption)); + } } } @Override public void checkCanRevokeTablePrivilege(TransactionId transactionId, Identity identity, AccessControlContext context, Privilege privilege, QualifiedObjectName tableName, PrestoPrincipal revokee, boolean grantOptionFor) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); - requireNonNull(privilege, "privilege is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_REVOKE_TABLE_PRIVILEGE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); + requireNonNull(privilege, "privilege is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRevokeTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), revokee, grantOptionFor)); + authorizationCheck(() -> systemAccessControl.get().checkCanRevokeTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), revokee, grantOptionFor)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanRevokeTablePrivilege(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, privilege, toSchemaTableName(tableName), revokee, grantOptionFor)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanRevokeTablePrivilege(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, privilege, toSchemaTableName(tableName), revokee, grantOptionFor)); + } } } @Override public void checkCanSetSystemSessionProperty(Identity identity, AccessControlContext context, String propertyName) { - requireNonNull(identity, "identity is null"); - requireNonNull(propertyName, "propertyName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SET_SYSTEM_SESSION_PROPERTY.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(propertyName, "propertyName is null"); - authorizationCheck(() -> systemAccessControl.get().checkCanSetSystemSessionProperty(identity, context, propertyName)); + authorizationCheck(() -> systemAccessControl.get().checkCanSetSystemSessionProperty(identity, context, propertyName)); + } } @Override public void checkCanSetCatalogSessionProperty(TransactionId transactionId, Identity identity, AccessControlContext context, String catalogName, String propertyName) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalogName is null"); - requireNonNull(propertyName, "propertyName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SET_CATALOG_SESSION_PROPERTY.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalogName is null"); + requireNonNull(propertyName, "propertyName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - authorizationCheck(() -> systemAccessControl.get().checkCanSetCatalogSessionProperty(identity, context, catalogName, propertyName)); + authorizationCheck(() -> systemAccessControl.get().checkCanSetCatalogSessionProperty(identity, context, catalogName, propertyName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanSetCatalogSessionProperty(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, propertyName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanSetCatalogSessionProperty(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, propertyName)); + } } } @Override public void checkCanSelectFromColumns(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName, Set columnOrSubfieldNames) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); - requireNonNull(columnOrSubfieldNames, "columnOrSubfieldNames is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SELECT_FROM_COLUMNS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); + requireNonNull(columnOrSubfieldNames, "columnOrSubfieldNames is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanSelectFromColumns( - identity, - context, - toCatalogSchemaTableName(tableName), - columnOrSubfieldNames.stream().map(subfield -> subfield.getRootName()).collect(toImmutableSet()))); + authorizationCheck(() -> systemAccessControl.get().checkCanSelectFromColumns( + identity, + context, + toCatalogSchemaTableName(tableName), + columnOrSubfieldNames.stream().map(subfield -> subfield.getRootName()).collect(toImmutableSet()))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanSelectFromColumns(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), columnOrSubfieldNames)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanSelectFromColumns(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), columnOrSubfieldNames)); + } } } @Override public void checkCanCreateRole(TransactionId transactionId, Identity identity, AccessControlContext context, String role, Optional grantor, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(role, "role is null"); - requireNonNull(grantor, "grantor is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_CREATE_ROLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(role, "role is null"); + requireNonNull(grantor, "grantor is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanCreateRole(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, role, grantor)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanCreateRole(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, role, grantor)); + } } } @Override public void checkCanDropRole(TransactionId transactionId, Identity identity, AccessControlContext context, String role, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(role, "role is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_DROP_ROLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(role, "role is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanDropRole(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, role)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanDropRole(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, role)); + } } } @Override public void checkCanGrantRoles(TransactionId transactionId, Identity identity, AccessControlContext context, Set roles, Set grantees, boolean withAdminOption, Optional grantor, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(roles, "roles is null"); - requireNonNull(grantees, "grantees is null"); - requireNonNull(grantor, "grantor is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_GRANT_ROLES.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(roles, "roles is null"); + requireNonNull(grantees, "grantees is null"); + requireNonNull(grantor, "grantor is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanGrantRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, roles, grantees, withAdminOption, grantor, catalogName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanGrantRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, roles, grantees, withAdminOption, grantor, catalogName)); + } } } @Override public void checkCanRevokeRoles(TransactionId transactionId, Identity identity, AccessControlContext context, Set roles, Set grantees, boolean adminOptionFor, Optional grantor, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(roles, "roles is null"); - requireNonNull(grantees, "grantees is null"); - requireNonNull(grantor, "grantor is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_REVOKE_ROLES.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(roles, "roles is null"); + requireNonNull(grantees, "grantees is null"); + requireNonNull(grantor, "grantor is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanRevokeRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, roles, grantees, adminOptionFor, grantor, catalogName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanRevokeRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, roles, grantees, adminOptionFor, grantor, catalogName)); + } } } @Override public void checkCanSetRole(TransactionId transactionId, Identity identity, AccessControlContext context, String role, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(role, "role is null"); - requireNonNull(catalogName, "catalog is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SET_ROLE.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(role, "role is null"); + requireNonNull(catalogName, "catalog is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanSetRole(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, role, catalogName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanSetRole(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, role, catalogName)); + } } } @Override public void checkCanShowRoles(TransactionId transactionId, Identity identity, AccessControlContext context, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SHOW_ROLES.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authenticationCheck(() -> entry.getAccessControl().checkCanShowRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, catalogName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authenticationCheck(() -> entry.getAccessControl().checkCanShowRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, catalogName)); + } } } @Override public void checkCanShowCurrentRoles(TransactionId transactionId, Identity identity, AccessControlContext context, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SHOW_CURRENT_ROLES.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authenticationCheck(() -> entry.getAccessControl().checkCanShowCurrentRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, catalogName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authenticationCheck(() -> entry.getAccessControl().checkCanShowCurrentRoles(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, catalogName)); + } } } @Override public void checkCanShowRoleGrants(TransactionId transactionId, Identity identity, AccessControlContext context, String catalogName) { - requireNonNull(identity, "identity is null"); - requireNonNull(catalogName, "catalogName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_SHOW_ROLE_GRANTS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(catalogName, "catalogName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); - if (entry != null) { - authenticationCheck(() -> entry.getAccessControl().checkCanShowRoleGrants(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, catalogName)); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); + if (entry != null) { + authenticationCheck(() -> entry.getAccessControl().checkCanShowRoleGrants(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(catalogName), context, catalogName)); + } } } @Override public void checkCanDropConstraint(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_DROP_CONSTRAINT.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropConstraint(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanDropConstraint(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanDropConstraint(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanDropConstraint(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } @Override public void checkCanAddConstraints(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { - requireNonNull(identity, "identity is null"); - requireNonNull(tableName, "tableName is null"); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.CHECK_CAN_ADD_CONSTRAINTS.getName(), skipSpan)) { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); - authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanAddConstraint(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.get().checkCanAddConstraint(identity, context, toCatalogSchemaTableName(tableName))); - CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); - if (entry != null) { - authorizationCheck(() -> entry.getAccessControl().checkCanAddConstraint(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanAddConstraint(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } } } private CatalogAccessControlEntry getConnectorAccessControl(TransactionId transactionId, String catalogName) { - return transactionManager.getOptionalCatalogMetadata(transactionId, catalogName) - .map(metadata -> connectorAccessControl.get(metadata.getConnectorId())) - .orElse(null); + try (BaseSpan ignored = scopedSpan("AccessControl." + TracingEnum.GET_CONNECTOR_ACCESS_CONTROL.getName(), skipSpan)) { + return transactionManager.getOptionalCatalogMetadata(transactionId, catalogName) + .map(metadata -> connectorAccessControl.get(metadata.getConnectorId())) + .orElse(null); + } } @Managed diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index 2e20fe521c428..24ab3dfb65b5e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -89,6 +89,7 @@ import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.planner.PlanFragmenter; import com.facebook.presto.sql.planner.PlanOptimizers; +import com.facebook.presto.telemetry.TelemetryResource; import com.facebook.presto.transaction.ForTransactionManager; import com.facebook.presto.transaction.InMemoryTransactionManager; import com.facebook.presto.transaction.TransactionManager; @@ -161,6 +162,7 @@ protected void setup(Binder binder) jsonCodecBinder(binder).bindJsonCodec(QueryResults.class); jsonCodecBinder(binder).bindJsonCodec(SelectedRole.class); jaxrsBinder(binder).bind(QueuedStatementResource.class); + jaxrsBinder(binder).bind(TelemetryResource.class); jaxrsBinder(binder).bind(ExecutingStatementResource.class); binder.bind(StatementHttpExecutionMBean.class).in(Scopes.SINGLETON); newExporter(binder).export(StatementHttpExecutionMBean.class).withGeneratedName(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java b/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java index 6fc8b9efbd667..fc0f7a829a9d9 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java +++ b/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java @@ -24,15 +24,10 @@ import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.security.SelectedRole; import com.facebook.presto.spi.session.ResourceEstimates; -import com.facebook.presto.spi.tracing.Tracer; -import com.facebook.presto.spi.tracing.TracerHandle; -import com.facebook.presto.spi.tracing.TracerProvider; import com.facebook.presto.sql.parser.ParsingException; import com.facebook.presto.sql.parser.ParsingOptions; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.parser.SqlParserOptions; -import com.facebook.presto.tracing.NoopTracerProvider; -import com.facebook.presto.tracing.TracingConfig; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -60,7 +55,6 @@ import java.util.Set; import static com.facebook.airlift.json.JsonCodec.jsonCodec; -import static com.facebook.presto.SystemSessionProperties.DISTRIBUTED_TRACING_MODE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CATALOG; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_INFO; import static com.facebook.presto.client.PrestoHeaders.PRESTO_CLIENT_TAGS; @@ -74,7 +68,6 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_SESSION_FUNCTION; import static com.facebook.presto.client.PrestoHeaders.PRESTO_SOURCE; import static com.facebook.presto.client.PrestoHeaders.PRESTO_TIME_ZONE; -import static com.facebook.presto.client.PrestoHeaders.PRESTO_TRACE_TOKEN; import static com.facebook.presto.client.PrestoHeaders.PRESTO_TRANSACTION_ID; import static com.facebook.presto.client.PrestoHeaders.PRESTO_USER; import static com.facebook.presto.server.security.ServletSecurityUtils.authorizedIdentity; @@ -105,7 +98,6 @@ public final class HttpRequestSessionContext private final List certificates; private final String source; - private final Optional traceToken; private final String userAgent; private final String remoteUserAddress; private final String timeZoneId; @@ -124,24 +116,22 @@ public final class HttpRequestSessionContext private final Map sessionFunctions; private final Optional sessionPropertyManager; - private final Optional tracer; private final RuntimeStats runtimeStats = new RuntimeStats(); public HttpRequestSessionContext(HttpServletRequest servletRequest, SqlParserOptions sqlParserOptions) { - this(servletRequest, sqlParserOptions, NoopTracerProvider.NOOP_TRACER_PROVIDER, Optional.empty()); + this(servletRequest, sqlParserOptions, Optional.empty()); } /** * @param servletRequest * @param sqlParserOptions - * @param tracerProvider This passed-in {@link TracerProvider} will only be used when isTracingEnabled() returns true. * @param sessionPropertyManager is used to provide with some default session values. In some scenarios we need * those default values even before session for a query is created. This is how we can get it at this * session context creation stage. * @throws WebApplicationException */ - public HttpRequestSessionContext(HttpServletRequest servletRequest, SqlParserOptions sqlParserOptions, TracerProvider tracerProvider, Optional sessionPropertyManager) + public HttpRequestSessionContext(HttpServletRequest servletRequest, SqlParserOptions sqlParserOptions, Optional sessionPropertyManager) throws WebApplicationException { catalog = trimEmptyToNull(servletRequest.getHeader(PRESTO_CATALOG)); @@ -220,26 +210,6 @@ else if (nameParts.size() == 2) { this.sessionPropertyManager = requireNonNull(sessionPropertyManager, "sessionPropertyManager is null"); Map requestHeaders = getRequestHeaders(servletRequest); - TracerHandle tracerHandle = tracerProvider.getHandleGenerator().apply(requestHeaders); - - if (isTracingEnabled()) { - this.tracer = Optional.of(requireNonNull(tracerProvider.getNewTracer(tracerHandle), "tracer is null")); - traceToken = Optional.ofNullable(this.tracer.get().getTracerId()); - } - else { - this.tracer = Optional.of(NoopTracerProvider.NOOP_TRACER); - - // If tunnel trace token is null, we expose the Presto tracing id. - // Otherwise we preserve the ability of trace token tunneling but - // still trace Presto internally for aggregation purposes. - String tunnelTraceId = trimEmptyToNull(servletRequest.getHeader(PRESTO_TRACE_TOKEN)); - if (tunnelTraceId != null) { - traceToken = Optional.of(tunnelTraceId); - } - else { - traceToken = Optional.ofNullable(tracerHandle.getTraceToken()); - } - } } private static Map getRequestHeaders(HttpServletRequest servletRequest) @@ -516,18 +486,6 @@ public Map getSessionFunctions() return sessionFunctions; } - @Override - public Optional getTraceToken() - { - return traceToken; - } - - @Override - public Optional getTracer() - { - return tracer; - } - @Override public RuntimeStats getRuntimeStats() { @@ -539,26 +497,6 @@ public RuntimeStats getRuntimeStats() * as highest priority. If client does not provide any session enabling property, we then take the system * default session value for determining if we should trace this query. */ - private boolean isTracingEnabled() - { - String clientValue = systemProperties.getOrDefault(DISTRIBUTED_TRACING_MODE, ""); - - // Client session setting overrides everything. - if (clientValue.equalsIgnoreCase(TracingConfig.DistributedTracingMode.ALWAYS_TRACE.name())) { - return true; - } - if (clientValue.equalsIgnoreCase(TracingConfig.DistributedTracingMode.NO_TRACE.name())) { - return false; - } - if (clientValue.equalsIgnoreCase(TracingConfig.DistributedTracingMode.SAMPLE_BASED.name())) { - return true; - } - - // Client not set, we then take system default value if ALWAYS_TRACE (SAMPLE_BASED disabled). If property manager not provided then false. - return sessionPropertyManager - .map(manager -> manager.decodeSystemPropertyValue(DISTRIBUTED_TRACING_MODE, null, String.class).equalsIgnoreCase(TracingConfig.DistributedTracingMode.ALWAYS_TRACE.name())) - .orElse(false); - } private Set parseClientTags(HttpServletRequest servletRequest) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/NoOpSessionSupplier.java b/presto-main/src/main/java/com/facebook/presto/server/NoOpSessionSupplier.java index a7717bda17c43..b1b822cf6c3fe 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/NoOpSessionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/server/NoOpSessionSupplier.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollectorFactory; import com.facebook.presto.spi.QueryId; +import com.facebook.presto.spi.telemetry.BaseSpan; import static com.facebook.presto.Session.SessionBuilder; @@ -26,13 +27,13 @@ public class NoOpSessionSupplier implements SessionSupplier { @Override - public Session createSession(QueryId queryId, SessionContext context, WarningCollectorFactory warningCollectorFactory) + public Session createSession(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, SessionContext context, WarningCollectorFactory warningCollectorFactory) { throw new UnsupportedOperationException(); } @Override - public SessionBuilder createSessionBuilder(QueryId queryId, SessionContext context, WarningCollectorFactory warningCollectorFactory) + public SessionBuilder createSessionBuilder(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, SessionContext context, WarningCollectorFactory warningCollectorFactory) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java index 274f90bed1204..6367a649dc774 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PluginManager.java @@ -50,7 +50,7 @@ import com.facebook.presto.spi.sql.planner.ExpressionOptimizerFactory; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.storage.TempStorageFactory; -import com.facebook.presto.spi.tracing.TracerProvider; +import com.facebook.presto.spi.telemetry.TelemetryFactory; import com.facebook.presto.spi.ttl.ClusterTtlProviderFactory; import com.facebook.presto.spi.ttl.NodeTtlFetcherFactory; import com.facebook.presto.sql.analyzer.AnalyzerProviderManager; @@ -58,7 +58,7 @@ import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.storage.TempStorageManager; -import com.facebook.presto.tracing.TracerProviderManager; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManager; import com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManager; import com.google.common.collect.ImmutableList; @@ -137,13 +137,13 @@ public class PluginManager private final AtomicBoolean pluginsLoaded = new AtomicBoolean(); private final ImmutableSet disabledConnectors; private final HistoryBasedPlanStatisticsManager historyBasedPlanStatisticsManager; - private final TracerProviderManager tracerProviderManager; private final AnalyzerProviderManager analyzerProviderManager; private final QueryPreparerProviderManager queryPreparerProviderManager; private final NodeStatusNotificationManager nodeStatusNotificationManager; private final ClientRequestFilterManager clientRequestFilterManager; private final PlanCheckerProviderManager planCheckerProviderManager; private final ExpressionOptimizerManager expressionOptimizerManager; + private final TracingManager tracingManager; @Inject public PluginManager( @@ -165,11 +165,11 @@ public PluginManager( NodeTtlFetcherManager nodeTtlFetcherManager, ClusterTtlProviderManager clusterTtlProviderManager, HistoryBasedPlanStatisticsManager historyBasedPlanStatisticsManager, - TracerProviderManager tracerProviderManager, NodeStatusNotificationManager nodeStatusNotificationManager, ClientRequestFilterManager clientRequestFilterManager, PlanCheckerProviderManager planCheckerProviderManager, - ExpressionOptimizerManager expressionOptimizerManager) + ExpressionOptimizerManager expressionOptimizerManager, + TracingManager tracingManager) { requireNonNull(nodeInfo, "nodeInfo is null"); requireNonNull(config, "config is null"); @@ -198,13 +198,13 @@ public PluginManager( this.clusterTtlProviderManager = requireNonNull(clusterTtlProviderManager, "clusterTtlProviderManager is null"); this.disabledConnectors = requireNonNull(config.getDisabledConnectors(), "disabledConnectors is null"); this.historyBasedPlanStatisticsManager = requireNonNull(historyBasedPlanStatisticsManager, "historyBasedPlanStatisticsManager is null"); - this.tracerProviderManager = requireNonNull(tracerProviderManager, "tracerProviderManager is null"); this.analyzerProviderManager = requireNonNull(analyzerProviderManager, "analyzerProviderManager is null"); this.queryPreparerProviderManager = requireNonNull(queryPreparerProviderManager, "queryPreparerProviderManager is null"); this.nodeStatusNotificationManager = requireNonNull(nodeStatusNotificationManager, "nodeStatusNotificationManager is null"); this.clientRequestFilterManager = requireNonNull(clientRequestFilterManager, "clientRequestFilterManager is null"); this.planCheckerProviderManager = requireNonNull(planCheckerProviderManager, "planCheckerProviderManager is null"); this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionManager is null"); + this.tracingManager = requireNonNull(tracingManager, "tracingManager is null"); } public void loadPlugins() @@ -355,11 +355,6 @@ public void installPlugin(Plugin plugin) historyBasedPlanStatisticsManager.addHistoryBasedPlanStatisticsProviderFactory(historyBasedPlanStatisticsProvider); } - for (TracerProvider tracerProvider : plugin.getTracerProviders()) { - log.info("Registering tracer provider %s", tracerProvider.getName()); - tracerProviderManager.addTracerProviderFactory(tracerProvider); - } - for (AnalyzerProvider analyzerProvider : plugin.getAnalyzerProviders()) { log.info("Registering analyzer provider %s", analyzerProvider.getType()); analyzerProviderManager.addAnalyzerProvider(analyzerProvider); @@ -379,6 +374,11 @@ public void installPlugin(Plugin plugin) log.info("Registering client request filter factory"); clientRequestFilterManager.registerClientRequestFilterFactory(clientRequestFilterFactory); } + + for (TelemetryFactory telemetryFactories : plugin.getTelemetryFactories()) { + log.info("Registering event listener %s", telemetryFactories.getName()); + tracingManager.addOpenTelemetryFactory(telemetryFactories); + } } public void installCoordinatorPlugin(CoordinatorPlugin plugin) diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 19250dd963eb7..3ac59267c88a7 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -59,7 +59,8 @@ import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.storage.TempStorageManager; import com.facebook.presto.storage.TempStorageModule; -import com.facebook.presto.tracing.TracerProviderManager; +import com.facebook.presto.telemetry.TelemetryModule; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManager; import com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManagerModule; import com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManager; @@ -129,6 +130,7 @@ public void run() new SmileModule(), new JaxrsModule(true), new MBeanModule(), + new TelemetryModule(), new JmxModule(), new JmxHttpModule(), new LogJmxModule(), @@ -173,6 +175,8 @@ public void run() injector.getInstance(Announcer.class), injector.getInstance(DriftServer.class)); + injector.getInstance(TracingManager.class).loadConfiguredOpenTelemetry(); + log.info("telemetry configs loaded"); injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers(); injector.getInstance(SessionPropertyDefaults.class).loadConfigurationManager(); injector.getInstance(ResourceGroupManager.class).loadConfigurationManager(); @@ -186,7 +190,6 @@ public void run() injector.getInstance(QueryPrerequisitesManager.class).loadQueryPrerequisites(); injector.getInstance(NodeTtlFetcherManager.class).loadNodeTtlFetcher(); injector.getInstance(ClusterTtlProviderManager.class).loadClusterTtlProvider(); - injector.getInstance(TracerProviderManager.class).loadTracerProvider(); injector.getInstance(NodeStatusNotificationManager.class).loadNodeStatusNotificationProvider(); injector.getInstance(GracefulShutdownHandler.class).loadNodeStatusNotification(); injector.getInstance(SessionPropertyManager.class).loadSessionPropertyProviders(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java b/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java index 260611d190033..ddbef1d64e0de 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java @@ -27,6 +27,7 @@ import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.spi.security.AuthorizedIdentity; import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.SqlEnvironmentConfig; import com.facebook.presto.transaction.TransactionManager; @@ -74,9 +75,9 @@ public QuerySessionSupplier( } @Override - public Session createSession(QueryId queryId, SessionContext context, WarningCollectorFactory warningCollectorFactory) + public Session createSession(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, SessionContext context, WarningCollectorFactory warningCollectorFactory) { - Session session = createSessionBuilder(queryId, context, warningCollectorFactory).build(); + Session session = createSessionBuilder(queryId, querySpan, rootSpan, context, warningCollectorFactory).build(); if (context.getTransactionId().isPresent()) { session = session.beginTransactionId(context.getTransactionId().get(), transactionManager, accessControl); } @@ -84,10 +85,12 @@ public Session createSession(QueryId queryId, SessionContext context, WarningCol } @Override - public SessionBuilder createSessionBuilder(QueryId queryId, SessionContext context, WarningCollectorFactory warningCollectorFactory) + public SessionBuilder createSessionBuilder(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, SessionContext context, WarningCollectorFactory warningCollectorFactory) { SessionBuilder sessionBuilder = Session.builder(sessionPropertyManager) .setQueryId(queryId) + .setQuerySpan(querySpan) + .setRootSpan(rootSpan) .setIdentity(authenticateIdentity(queryId, context)) .setSource(context.getSource()) .setCatalog(context.getCatalog()) @@ -96,9 +99,7 @@ public SessionBuilder createSessionBuilder(QueryId queryId, SessionContext conte .setUserAgent(context.getUserAgent()) .setClientInfo(context.getClientInfo()) .setClientTags(context.getClientTags()) - .setTraceToken(context.getTraceToken()) .setResourceEstimates(context.getResourceEstimates()) - .setTracer(context.getTracer()) .setRuntimeStats(context.getRuntimeStats()); if (forcedSessionTimeZone.isPresent()) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index 3071bd0e5d0f3..86e6565d8f66a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -218,8 +218,6 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.statusservice.NodeStatusService; -import com.facebook.presto.tracing.TracerProviderManager; -import com.facebook.presto.tracing.TracingConfig; import com.facebook.presto.transaction.TransactionManagerConfig; import com.facebook.presto.type.TypeDeserializer; import com.facebook.presto.util.FinalizerService; @@ -797,10 +795,6 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon // cleanup binder.bind(ExecutorCleanup.class).in(Scopes.SINGLETON); - // Distributed tracing - configBinder(binder).bindConfig(TracingConfig.class); - binder.bind(TracerProviderManager.class).in(Scopes.SINGLETON); - //Optional Status Detector newOptionalBinder(binder, NodeStatusService.class); binder.bind(NodeStatusNotificationManager.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java b/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java index d40bb15482377..a12af4f1d173e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java +++ b/presto-main/src/main/java/com/facebook/presto/server/SessionContext.java @@ -20,7 +20,6 @@ import com.facebook.presto.spi.security.AuthorizedIdentity; import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.session.ResourceEstimates; -import com.facebook.presto.spi.tracing.Tracer; import com.google.common.collect.ImmutableList; import javax.annotation.Nullable; @@ -72,8 +71,6 @@ default List getCertificates() @Nullable String getLanguage(); - Optional getTracer(); - Map getSystemProperties(); Map> getCatalogSessionProperties(); @@ -82,8 +79,6 @@ default List getCertificates() Optional getTransactionId(); - Optional getTraceToken(); - boolean supportClientTransaction(); Map getSessionFunctions(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/SessionSupplier.java b/presto-main/src/main/java/com/facebook/presto/server/SessionSupplier.java index 240439d67ce09..53c8461b2bd8f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/SessionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/server/SessionSupplier.java @@ -16,12 +16,13 @@ import com.facebook.presto.Session; import com.facebook.presto.execution.warnings.WarningCollectorFactory; import com.facebook.presto.spi.QueryId; +import com.facebook.presto.spi.telemetry.BaseSpan; import static com.facebook.presto.Session.SessionBuilder; public interface SessionSupplier { - Session createSession(QueryId queryId, SessionContext context, WarningCollectorFactory warningCollectorFactory); + Session createSession(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, SessionContext context, WarningCollectorFactory warningCollectorFactory); - SessionBuilder createSessionBuilder(QueryId queryId, SessionContext context, WarningCollectorFactory warningCollectorFactory); + SessionBuilder createSessionBuilder(QueryId queryId, BaseSpan querySpan, BaseSpan rootSpan, SessionContext context, WarningCollectorFactory warningCollectorFactory); } diff --git a/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java b/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java index 17dc3450790e7..fea013873d7b6 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/TaskResource.java @@ -28,8 +28,11 @@ import com.facebook.presto.metadata.HandleResolver; import com.facebook.presto.metadata.MetadataUpdates; import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.operator.TaskStats; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.Duration; @@ -55,6 +58,7 @@ import javax.ws.rs.core.UriInfo; import java.util.List; +import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; @@ -71,6 +75,9 @@ import static com.facebook.presto.server.TaskResourceUtils.convertToThriftTaskInfo; import static com.facebook.presto.server.TaskResourceUtils.isThriftRequest; import static com.facebook.presto.server.security.RoleType.INTERNAL; +import static com.facebook.presto.telemetry.TracingManager.getSpan; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; +import static com.facebook.presto.telemetry.TracingManager.setAttributes; import static com.facebook.presto.util.TaskUtils.randomizeWaitTime; import static com.google.common.collect.Iterables.transform; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; @@ -131,20 +138,27 @@ public List getAllTaskInfo(@Context UriInfo uriInfo) @Path("{taskId}") @Consumes({APPLICATION_JSON, APPLICATION_JACKSON_SMILE}) @Produces({APPLICATION_JSON, APPLICATION_JACKSON_SMILE}) - public Response createOrUpdateTask(@PathParam("taskId") TaskId taskId, TaskUpdateRequest taskUpdateRequest, @Context UriInfo uriInfo) + public Response createOrUpdateTask(@PathParam("taskId") TaskId taskId, TaskUpdateRequest taskUpdateRequest, @Context UriInfo uriInfo, @HeaderParam("traceparent") String traceParent) { requireNonNull(taskUpdateRequest, "taskUpdateRequest is null"); - Session session = taskUpdateRequest.getSession().toSession(sessionPropertyManager, taskUpdateRequest.getExtraCredentials()); - TaskInfo taskInfo = taskManager.updateTask(session, - taskId, - taskUpdateRequest.getFragment().map(planFragmentCodec::fromBytes), - taskUpdateRequest.getSources(), - taskUpdateRequest.getOutputIds(), - taskUpdateRequest.getTableWriteInfo()); + BaseSpan span = getSpan(traceParent, "POST /v1/task/{taskId}"); - if (shouldSummarize(uriInfo)) { - taskInfo = taskInfo.summarize(); + TaskInfo taskInfo; + + try (BaseSpan ignored = scopedSpan(span)) { + Session session = taskUpdateRequest.getSession().toSession(sessionPropertyManager, taskUpdateRequest.getExtraCredentials()); + taskInfo = taskManager.updateTask(session, + taskId, + taskUpdateRequest.getFragment().map(planFragmentCodec::fromBytes), + taskUpdateRequest.getSources(), + taskUpdateRequest.getOutputIds(), + taskUpdateRequest.getTableWriteInfo(), + span); + + if (shouldSummarize(uriInfo)) { + taskInfo = taskInfo.summarize(); + } } return Response.ok().entity(taskInfo).build(); @@ -160,10 +174,13 @@ public void getTaskInfo( @HeaderParam(PRESTO_MAX_WAIT) Duration maxWait, @Context UriInfo uriInfo, @Context HttpHeaders httpHeaders, - @Suspended AsyncResponse asyncResponse) + @Suspended AsyncResponse asyncResponse, + @HeaderParam("traceparent") String traceParent) { requireNonNull(taskId, "taskId is null"); + BaseSpan span = getSpan(traceParent, "GET /v1/task/{taskId}"); + boolean isThriftRequest = isThriftRequest(httpHeaders); if (currentState == null || maxWait == null) { @@ -201,6 +218,10 @@ public void getTaskInfo( Duration timeout = new Duration(waitTime.toMillis() + ADDITIONAL_WAIT_TIME.toMillis(), MILLISECONDS); bindAsyncResponse(asyncResponse, futureTaskInfo, responseExecutor) .withTimeout(timeout); + + if (!Objects.isNull(span)) { + span.end(); + } } @GET @@ -212,10 +233,13 @@ public void getTaskStatus( @HeaderParam(PRESTO_CURRENT_STATE) TaskState currentState, @HeaderParam(PRESTO_MAX_WAIT) Duration maxWait, @Context UriInfo uriInfo, - @Suspended AsyncResponse asyncResponse) + @Suspended AsyncResponse asyncResponse, + @HeaderParam("traceparent") String traceParent) { requireNonNull(taskId, "taskId is null"); + BaseSpan span = getSpan(traceParent, "GET /v1/task/{taskId}/status"); + if (currentState == null || maxWait == null) { TaskStatus taskStatus = taskManager.getTaskStatus(taskId); asyncResponse.resume(taskStatus); @@ -236,15 +260,26 @@ public void getTaskStatus( Duration timeout = new Duration(waitTime.toMillis() + ADDITIONAL_WAIT_TIME.toMillis(), MILLISECONDS); bindAsyncResponse(asyncResponse, futureTaskStatus, responseExecutor) .withTimeout(timeout); + + if (!Objects.isNull(span)) { + span.end(); + } } @POST @Path("{taskId}/metadataresults") @Consumes({APPLICATION_JSON, APPLICATION_JACKSON_SMILE}) - public Response updateMetadataResults(@PathParam("taskId") TaskId taskId, MetadataUpdates metadataUpdates, @Context UriInfo uriInfo) + public Response updateMetadataResults(@PathParam("taskId") TaskId taskId, MetadataUpdates metadataUpdates, @Context UriInfo uriInfo, @HeaderParam("traceparent") String traceParent) { requireNonNull(metadataUpdates, "metadataUpdates is null"); + + BaseSpan span = getSpan(traceParent, "POST /v1/task/{taskId}/metadataresults"); + taskManager.updateMetadataResults(taskId, metadataUpdates); + + if (!Objects.isNull(span)) { + span.end(); + } return Response.ok().build(); } @@ -256,16 +291,25 @@ public TaskInfo deleteTask( @PathParam("taskId") TaskId taskId, @QueryParam("abort") @DefaultValue("true") boolean abort, @Context UriInfo uriInfo, - @Context HttpHeaders httpHeaders) + @Context HttpHeaders httpHeaders, + @HeaderParam("traceparent") String traceParent) { requireNonNull(taskId, "taskId is null"); TaskInfo taskInfo; + BaseSpan span = getSpan(traceParent, "DELETE /v1/task/{taskId}"); + if (abort) { taskInfo = taskManager.abortTask(taskId); + if (Objects.nonNull(span)) { + setAttributes(span, ImmutableMap.of("status", "aborted")); + } } else { taskInfo = taskManager.cancelTask(taskId); + if (Objects.nonNull(span)) { + setAttributes(span, ImmutableMap.of("status", "cancelled")); + } } if (shouldSummarize(uriInfo)) { @@ -276,6 +320,20 @@ public TaskInfo deleteTask( taskInfo = convertToThriftTaskInfo(taskInfo, connectorTypeSerdeManager, handleResolver); } + String taskStatus = taskInfo.getTaskStatus().getState().toString(); + String tskId = String.valueOf(taskId.getId()); + String nodeId = taskInfo.getNodeId(); + TaskStats taskStats = taskInfo.getStats(); + String createTime = taskStats.getCreateTime().toString(); + String endTime = taskStats.getEndTime() != null ? taskStats.getEndTime().toString() : null; + String noOfSplits = String.valueOf(taskStats.getTotalDrivers()); + String lastHeartbeat = taskInfo.getLastHeartbeat().toString(); + String outputBufferInfo = taskInfo.getOutputBuffers().toString(); + + if (Objects.nonNull(span)) { + setAttributes(span, ImmutableMap.of("task status", taskStatus, "taskId", tskId, "node id", nodeId, "create time", createTime, "end time", endTime, "no. of splits", noOfSplits, "last heartbeat", lastHeartbeat, "output buffer state", outputBufferInfo)); + span.end(); + } return taskInfo; } @@ -284,23 +342,37 @@ public TaskInfo deleteTask( public void acknowledgeResults( @PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, - @PathParam("token") final long token) + @PathParam("token") final long token, + @HeaderParam("traceparent") String traceParent) { requireNonNull(taskId, "taskId is null"); requireNonNull(bufferId, "bufferId is null"); + BaseSpan span = getSpan(traceParent, "GET /v1/task/{taskId}/results/{bufferId}/{token}/acknowledge"); + taskManager.acknowledgeTaskResults(taskId, bufferId, token); + + if (!Objects.isNull(span)) { + span.end(); + } } @HEAD @Path("{taskId}/results/{bufferId}") public Response taskResultsHeaders( @PathParam("taskId") TaskId taskId, - @PathParam("bufferId") OutputBufferId bufferId) + @PathParam("bufferId") OutputBufferId bufferId, + @HeaderParam("traceparent") String traceParent) { requireNonNull(taskId, "taskId is null"); requireNonNull(bufferId, "bufferId is null"); + BaseSpan span = getSpan(traceParent, "HEAD /v1/task/{taskId}/results/{bufferId}"); + + if (!Objects.isNull(span)) { + span.end(); + } + OutputBufferInfo outputBufferInfo = taskManager.getOutputBufferInfo(taskId); return outputBufferInfo.getBuffers().stream() .filter(bufferInfo -> bufferInfo.getBufferId().equals(bufferId)) @@ -322,31 +394,50 @@ public Response taskResultsHeaders( public Response taskResultsHeaders( @PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, - @PathParam("token") final long token) + @PathParam("token") final long token, + @HeaderParam("traceparent") String traceParent) { + BaseSpan span = getSpan(traceParent, "HEAD /v1/task/{taskId}/results/{bufferId}/{token}"); + + if (!Objects.isNull(span)) { + span.end(); + } + taskManager.acknowledgeTaskResults(taskId, bufferId, token); - return taskResultsHeaders(taskId, bufferId); + return taskResultsHeaders(taskId, bufferId, traceParent); } @DELETE @Path("{taskId}/results/{bufferId}") @Produces(APPLICATION_JSON) - public void abortResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @Context UriInfo uriInfo) + public void abortResults(@PathParam("taskId") TaskId taskId, @PathParam("bufferId") OutputBufferId bufferId, @Context UriInfo uriInfo, @HeaderParam("traceparent") String traceParent) { requireNonNull(taskId, "taskId is null"); requireNonNull(bufferId, "bufferId is null"); + BaseSpan span = getSpan(traceParent, "DELETE v1/task/{taskId}/results/{bufferId}"); + taskManager.abortTaskResults(taskId, bufferId); + + if (!Objects.isNull(span)) { + span.end(); + } } @DELETE @Path("{taskId}/remote-source/{remoteSourceTaskId}") - public void removeRemoteSource(@PathParam("taskId") TaskId taskId, @PathParam("remoteSourceTaskId") TaskId remoteSourceTaskId) + public void removeRemoteSource(@PathParam("taskId") TaskId taskId, @PathParam("remoteSourceTaskId") TaskId remoteSourceTaskId, @HeaderParam("traceparent") String traceParent) { requireNonNull(taskId, "taskId is null"); requireNonNull(remoteSourceTaskId, "remoteSourceTaskId is null"); + BaseSpan span = getSpan(traceParent, "DELETE /v1/task/{taskId}/remote-source/{remoteSourceTaskId}"); + taskManager.removeRemoteSource(taskId, remoteSourceTaskId); + + if (!Objects.isNull(span)) { + span.end(); + } } private static boolean shouldSummarize(UriInfo uriInfo) diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java index c944f0cd468b6..126f1672038f6 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/Query.java @@ -41,7 +41,6 @@ import com.facebook.presto.spi.page.PagesSerde; import com.facebook.presto.spi.page.SerializedPage; import com.facebook.presto.spi.security.SelectedRole; -import com.facebook.presto.spi.tracing.Tracer; import com.facebook.presto.transaction.TransactionInfo; import com.facebook.presto.transaction.TransactionManager; import com.google.common.collect.ImmutableList; @@ -256,13 +255,6 @@ public boolean isSlugValid(String slug) return this.slug.equals(slug); } - public Tracer getTracer() - { - Optional tracer = session.getTracer(); - checkArgument(tracer.isPresent(), "tracer is not present"); - return tracer.get(); - } - public synchronized Optional getSetCatalog() { return setCatalog; diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/QueuedStatementResource.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/QueuedStatementResource.java index 82b4b55cbb459..cc03bad79a5db 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/QueuedStatementResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/QueuedStatementResource.java @@ -18,6 +18,7 @@ import com.facebook.presto.client.QueryError; import com.facebook.presto.client.QueryResults; import com.facebook.presto.common.ErrorCode; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.dispatcher.DispatchExecutor; import com.facebook.presto.dispatcher.DispatchInfo; import com.facebook.presto.dispatcher.DispatchManager; @@ -28,8 +29,10 @@ import com.facebook.presto.server.SessionContext; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.parser.SqlParserOptions; -import com.facebook.presto.tracing.TracerProviderManager; +import com.facebook.presto.telemetry.TracingManager; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Ordering; import com.google.common.util.concurrent.ListenableFuture; @@ -125,7 +128,6 @@ public class QueuedStatementResource private final boolean nestedDataSerializationEnabled; private final SqlParserOptions sqlParserOptions; - private final TracerProviderManager tracerProviderManager; private final SessionPropertyManager sessionPropertyManager; // We may need some system default session property values at early query stage even before session is created. private final QueryBlockingRateLimiter queryRateLimiter; @@ -137,7 +139,6 @@ public QueuedStatementResource( ExecutingQueryResponseProvider executingQueryResponseProvider, SqlParserOptions sqlParserOptions, ServerConfig serverConfig, - TracerProviderManager tracerProviderManager, SessionPropertyManager sessionPropertyManager, QueryBlockingRateLimiter queryRateLimiter) { @@ -149,7 +150,6 @@ public QueuedStatementResource( this.responseExecutor = requireNonNull(executor, "responseExecutor is null").getExecutor(); this.timeoutExecutor = requireNonNull(executor, "timeoutExecutor is null").getScheduledExecutor(); - this.tracerProviderManager = requireNonNull(tracerProviderManager, "tracerProviderManager is null"); this.sessionPropertyManager = sessionPropertyManager; this.queryRateLimiter = requireNonNull(queryRateLimiter, "queryRateLimiter is null"); @@ -216,7 +216,6 @@ public Response postStatement( SessionContext sessionContext = new HttpRequestSessionContext( servletRequest, sqlParserOptions, - tracerProviderManager.getTracerProvider(), Optional.of(sessionPropertyManager)); Query query = new Query(statement, sessionContext, dispatchManager, executingQueryResponseProvider, 0); queries.put(query.getQueryId(), query); @@ -264,7 +263,6 @@ public Response putStatement( SessionContext sessionContext = new HttpRequestSessionContext( servletRequest, sqlParserOptions, - tracerProviderManager.getTracerProvider(), Optional.of(sessionPropertyManager)); Query attemptedQuery = new Query(statement, sessionContext, dispatchManager, executingQueryResponseProvider, 0, queryId, slug); Query query = queries.computeIfAbsent(queryId, unused -> attemptedQuery); @@ -534,7 +532,12 @@ private ListenableFuture waitForDispatched() // if query submission has not finished, wait for it to finish synchronized (this) { if (querySubmissionFuture == null) { - querySubmissionFuture = dispatchManager.createQuery(queryId, slug, retryCount, sessionContext, query); + //start tracing with root span for POST /v1/statement endpoint + BaseSpan rootSpan = TracingManager.getRootSpan(); + BaseSpan querySpan = TracingManager.getSpan(rootSpan, TracingEnum.QUERY.getName(), ImmutableMap.of("QUERY_ID", queryId.toString())); + + //propagate root and query spans through session + querySubmissionFuture = dispatchManager.createQuery(queryId, querySpan, rootSpan, slug, retryCount, sessionContext, query); } if (!querySubmissionFuture.isDone()) { return querySubmissionFuture; diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java index 38314331e49fe..f19441f96f34a 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java @@ -33,12 +33,15 @@ import com.facebook.presto.server.thrift.ThriftHttpResponseHandler; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.telemetry.BaseSpan; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.Duration; import io.netty.channel.EventLoop; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; @@ -79,6 +82,7 @@ class ContinuousTaskStatusFetcher private final boolean thriftTransportEnabled; private final Protocol thriftProtocol; private long currentRequestStartNanos; + private final BaseSpan remoteTaskSpan; private boolean running; private ListenableFuture> future; @@ -95,7 +99,8 @@ public ContinuousTaskStatusFetcher( RemoteTaskStats stats, boolean binaryTransportEnabled, boolean thriftTransportEnabled, - Protocol thriftProtocol) + Protocol thriftProtocol, + BaseSpan remoteTaskSpan) { requireNonNull(initialTaskStatus, "initialTaskStatus is null"); @@ -114,6 +119,8 @@ public ContinuousTaskStatusFetcher( this.binaryTransportEnabled = binaryTransportEnabled; this.thriftTransportEnabled = thriftTransportEnabled; this.thriftProtocol = requireNonNull(thriftProtocol, "thriftProtocol is null"); + + this.remoteTaskSpan = remoteTaskSpan; } public void start() @@ -179,6 +186,12 @@ else if (binaryTransportEnabled) { responseHandler = createAdaptingJsonResponseHandler((JsonCodec) taskStatusCodec); } + Map headersMap = TracingManager.getHeadersMap(remoteTaskSpan); + + for (Map.Entry entry : headersMap.entrySet()) { + requestBuilder.addHeader(entry.getKey(), entry.getValue()); + } + Request request = requestBuilder.setUri(uriBuilderFrom(taskStatus.getSelf()).appendPath("status").build()) .setHeader(PRESTO_CURRENT_STATE, taskStatus.getState().toString()) .setHeader(PRESTO_MAX_WAIT, refreshMaxWait.toString()) diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java index c773d887edcf7..3a21271ba97e5 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTask.java @@ -28,6 +28,7 @@ import com.facebook.airlift.stats.DecayCounter; import com.facebook.drift.transport.netty.codec.Protocol; import com.facebook.presto.Session; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.connector.ConnectorTypeSerdeManager; import com.facebook.presto.execution.FutureStateChange; import com.facebook.presto.execution.Lifespan; @@ -61,10 +62,13 @@ import com.facebook.presto.spi.SplitWeight; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.base.Ticker; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.collect.ObjectArrays; @@ -160,6 +164,8 @@ public final class HttpRemoteTask private static final ThreadMXBean THREAD_MX_BEAN = (ThreadMXBean) ManagementFactory.getThreadMXBean(); private final TaskId taskId; + private final BaseSpan stageSpan; + private final BaseSpan span; private final URI taskLocation; private final URI remoteTaskLocation; @@ -266,7 +272,8 @@ public static HttpRemoteTask createHttpRemoteTask( HandleResolver handleResolver, ConnectorTypeSerdeManager connectorTypeSerdeManager, SchedulerStatsTracker schedulerStatsTracker, - HttpRemoteTaskFactory.SafeEventLoop taskEventLoop) + HttpRemoteTaskFactory.SafeEventLoop taskEventLoop, + BaseSpan stageSpan) { HttpRemoteTask task = new HttpRemoteTask(session, taskId, @@ -302,7 +309,8 @@ public static HttpRemoteTask createHttpRemoteTask( handleResolver, connectorTypeSerdeManager, schedulerStatsTracker, - taskEventLoop); + taskEventLoop, + stageSpan); task.initialize(); return task; } @@ -341,7 +349,8 @@ private HttpRemoteTask(Session session, HandleResolver handleResolver, ConnectorTypeSerdeManager connectorTypeSerdeManager, SchedulerStatsTracker schedulerStatsTracker, - HttpRemoteTaskFactory.SafeEventLoop taskEventLoop) + HttpRemoteTaskFactory.SafeEventLoop taskEventLoop, + BaseSpan stageSpan) { requireNonNull(session, "session is null"); requireNonNull(taskId, "taskId is null"); @@ -371,6 +380,8 @@ private HttpRemoteTask(Session session, this.taskEventLoop = taskEventLoop; this.taskId = taskId; + this.stageSpan = stageSpan; + this.span = TracingManager.getSpan(stageSpan, TracingEnum.REMOTE_TASK.getName(), ImmutableMap.of("QUERY_ID", taskId.getQueryId().toString(), "STAGE_ID", taskId.getStageId().toString(), "TASK_ID", taskId.toString())); this.taskLocation = location; this.remoteTaskLocation = remoteLocation; this.session = session; @@ -441,7 +452,8 @@ private HttpRemoteTask(Session session, stats, binaryTransportEnabled, thriftTransportEnabled, - thriftProtocol); + thriftProtocol, + span); this.taskInfoFetcher = new TaskInfoFetcher( this::failTask, @@ -462,7 +474,8 @@ private HttpRemoteTask(Session session, queryManager, handleResolver, connectorTypeSerdeManager, - thriftProtocol); + thriftProtocol, + span); } // this is a separate method to ensure that the `this` reference is not leaked during construction @@ -479,6 +492,9 @@ private void initialize() updateTaskStats(); updateSplitQueueSpace(); } + if (Objects.nonNull(span) && state.isDone()) { + span.end(); + } }); updateTaskStats(); @@ -1000,11 +1016,19 @@ private void sendUpdate() } } + //extract current context and pass it to worker nodes by injecting in http header for some of the TaskResource endpoints + Map headersMap = TracingManager.getHeadersMap(stageSpan); + HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus); - Request request = setContentTypeHeaders(binaryTransportEnabled, preparePost()) + Request.Builder requestBuilder = setContentTypeHeaders(binaryTransportEnabled, preparePost()) .setUri(uriBuilder.build()) - .setBodyGenerator(createStaticBodyGenerator(taskUpdateRequestJson)) - .build(); + .setBodyGenerator(createStaticBodyGenerator(taskUpdateRequestJson)); + + for (Map.Entry entry : headersMap.entrySet()) { + requestBuilder.addHeader(entry.getKey(), entry.getValue()); + } + + Request request = requestBuilder.build(); ResponseHandler responseHandler; if (binaryTransportEnabled) { @@ -1071,9 +1095,17 @@ public void cancel() return; } + //extract remote-write span context and pass it to worker nodes on GET /v1/task/{taskId}/status endpoint + Map headersMap = TracingManager.getHeadersMap(span); + // send cancel to task and ignore response HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus).addParameter("abort", "false"); Request.Builder builder = setContentTypeHeaders(binaryTransportEnabled, prepareDelete()); + + for (Map.Entry entry : headersMap.entrySet()) { + builder.addHeader(entry.getKey(), entry.getValue()); + } + if (taskInfoThriftTransportEnabled) { builder = ThriftRequestUtils.prepareThriftDelete(thriftProtocol); } diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java index ea153916e803b..4bd497a8e7549 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java @@ -46,6 +46,7 @@ import com.facebook.presto.server.InternalCommunicationConfig; import com.facebook.presto.server.TaskUpdateRequest; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -205,7 +206,8 @@ public RemoteTask createRemoteTask( NodeTaskMap.NodeStatsTracker nodeStatsTracker, boolean summarizeTaskInfo, TableWriteInfo tableWriteInfo, - SchedulerStatsTracker schedulerStatsTracker) + SchedulerStatsTracker schedulerStatsTracker, + BaseSpan stageSpan) { return HttpRemoteTask.createHttpRemoteTask( session, @@ -242,7 +244,8 @@ public RemoteTask createRemoteTask( handleResolver, connectorTypeSerdeManager, schedulerStatsTracker, - (SafeEventLoop) eventLoopGroup.next()); + (SafeEventLoop) eventLoopGroup.next(), + stageSpan); } /*** diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java index e99fd90f55bd5..f070e178e8178 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/TaskInfoFetcher.java @@ -40,6 +40,8 @@ import com.facebook.presto.server.SimpleHttpResponseHandler; import com.facebook.presto.server.smile.BaseResponse; import com.facebook.presto.server.thrift.ThriftHttpResponseHandler; +import com.facebook.presto.spi.telemetry.BaseSpan; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -47,6 +49,7 @@ import io.netty.channel.EventLoop; import java.net.URI; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; @@ -90,6 +93,7 @@ public class TaskInfoFetcher private final RequestErrorTracker errorTracker; private final boolean summarizeTaskInfo; + private final BaseSpan remoteTaskSpan; private long currentRequestStartNanos; private final RemoteTaskStats stats; @@ -127,7 +131,8 @@ public TaskInfoFetcher( QueryManager queryManager, HandleResolver handleResolver, ConnectorTypeSerdeManager connectorTypeSerdeManager, - Protocol thriftProtocol) + Protocol thriftProtocol, + BaseSpan remoteTaskSpan) { requireNonNull(initialTask, "initialTask is null"); @@ -144,6 +149,7 @@ public TaskInfoFetcher( this.errorTracker = taskRequestErrorTracker(taskId, initialTask.getTaskStatus().getSelf(), maxErrorDuration, taskEventLoop, "getting info for task"); this.summarizeTaskInfo = summarizeTaskInfo; + this.remoteTaskSpan = remoteTaskSpan; this.taskEventLoop = requireNonNull(taskEventLoop, "taskEventLoop is null"); this.httpClient = requireNonNull(httpClient, "httpClient is null"); @@ -285,6 +291,12 @@ else if (isBinaryTransportEnabled) { .setHeader(PRESTO_MAX_WAIT, taskInfoRefreshMaxWait.toString()); } + Map headersMap = TracingManager.getHeadersMap(remoteTaskSpan); + + for (Map.Entry entry : headersMap.entrySet()) { + requestBuilder.addHeader(entry.getKey(), entry.getValue()); + } + Request request = requestBuilder.setUri(uri).build(); errorTracker.startRequest(); future = httpClient.executeAsync(request, responseHandler); @@ -404,11 +416,19 @@ private void sendMetadataUpdates(MetadataUpdates results) return; } + //for POST {taskId}/metadataresults endpoint + Map headersMap = TracingManager.getHeadersMap(remoteTaskSpan); + byte[] metadataUpdatesJson = metadataUpdatesCodec.toBytes(results); - Request request = setContentTypeHeaders(isBinaryTransportEnabled, preparePost()) + Request.Builder requestBuilder = setContentTypeHeaders(isBinaryTransportEnabled, preparePost()) .setUri(uriBuilderFrom(taskStatus.getSelf()).appendPath("metadataresults").build()) - .setBodyGenerator(createStaticBodyGenerator(metadataUpdatesJson)) - .build(); + .setBodyGenerator(createStaticBodyGenerator(metadataUpdatesJson)); + + for (Map.Entry entry : headersMap.entrySet()) { + requestBuilder.addHeader(entry.getKey(), entry.getValue()); + } + + Request request = requestBuilder.build(); errorTracker.startRequest(); metadataUpdateFuture = httpClient.executeAsync(request, new ResponseHandler() diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index a34b46003ad89..68135b6725c1b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -82,10 +82,12 @@ import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.storage.TempStorageManager; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.testing.ProcedureTester; import com.facebook.presto.testing.TestingAccessControlManager; import com.facebook.presto.testing.TestingEventListenerManager; import com.facebook.presto.testing.TestingTempStorageManager; +import com.facebook.presto.testing.TestingTracingManager; import com.facebook.presto.testing.TestingWarningCollectorModule; import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManagerModule; @@ -184,6 +186,7 @@ public class TestingPrestoServer private final PlanCheckerProviderManager planCheckerProviderManager; private final NodeManager pluginNodeManager; private final ClientRequestFilterManager clientRequestFilterManager; + private final TestingTracingManager testingTracingManager; public static class TestShutdownAction implements ShutdownAction @@ -212,6 +215,12 @@ public synchronized boolean isShutdown() } } + public TestingPrestoServer(Map properties) + throws Exception + { + this(true, properties, null, null, new SqlParserOptions(), ImmutableList.of()); + } + public TestingPrestoServer() throws Exception { @@ -319,6 +328,7 @@ public TestingPrestoServer( .add(new ClusterTtlProviderManagerModule()) .add(new ClientRequestFilterModule()) .add(binder -> { + binder.bind(TracingManager.class).to(TestingTracingManager.class).in(Scopes.SINGLETON); binder.bind(TestingAccessControlManager.class).in(Scopes.SINGLETON); binder.bind(TestingEventListenerManager.class).in(Scopes.SINGLETON); binder.bind(TestingTempStorageManager.class).in(Scopes.SINGLETON); @@ -394,6 +404,7 @@ public TestingPrestoServer( clusterStateProvider = null; planCheckerProviderManager = injector.getInstance(PlanCheckerProviderManager.class); expressionManager.loadExpressionOptimizerFactories(); + testingTracingManager = (TestingTracingManager) injector.getInstance(TracingManager.class); } else if (resourceManager) { dispatchManager = null; @@ -406,6 +417,7 @@ else if (resourceManager) { eventListenerManager = ((TestingEventListenerManager) injector.getInstance(EventListenerManager.class)); clusterStateProvider = injector.getInstance(ResourceManagerClusterStateProvider.class); planCheckerProviderManager = null; + testingTracingManager = (TestingTracingManager) injector.getInstance(TracingManager.class); } else if (coordinatorSidecar) { dispatchManager = null; @@ -418,6 +430,7 @@ else if (coordinatorSidecar) { eventListenerManager = null; clusterStateProvider = null; planCheckerProviderManager = null; + testingTracingManager = null; } else if (catalogServer) { dispatchManager = null; @@ -430,6 +443,7 @@ else if (catalogServer) { eventListenerManager = null; clusterStateProvider = null; planCheckerProviderManager = null; + testingTracingManager = null; } else { dispatchManager = null; @@ -442,6 +456,7 @@ else if (catalogServer) { eventListenerManager = null; clusterStateProvider = null; planCheckerProviderManager = null; + testingTracingManager = null; } localMemoryManager = injector.getInstance(LocalMemoryManager.class); nodeManager = injector.getInstance(InternalNodeManager.class); @@ -667,6 +682,11 @@ public NodeManager getPluginNodeManager() return pluginNodeManager; } + public TestingTracingManager getTestingTracingManager() + { + return testingTracingManager; + } + public NodePartitioningManager getNodePartitioningManager() { return nodePartitioningManager; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java index 7356b7d464acf..b6c15e7fc219f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/Optimizer.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.common.TelemetryConfig; import com.facebook.presto.cost.CachingCostProvider; import com.facebook.presto.cost.CachingStatsProvider; import com.facebook.presto.cost.CostCalculator; @@ -31,6 +32,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.SemiJoinNode; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.PlannerUtils; import com.facebook.presto.sql.planner.TypeProvider; @@ -39,9 +41,12 @@ import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; import com.facebook.presto.sql.planner.optimizations.PlanOptimizerResult; import com.facebook.presto.sql.planner.sanity.PlanChecker; +import com.facebook.presto.telemetry.TracingManager; +import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static com.facebook.presto.SystemSessionProperties.getQueryAnalyzerTimeout; import static com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery; @@ -53,6 +58,8 @@ import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.OptimizerRuntimeTrackUtil.getOptimizerNameForLog; import static com.facebook.presto.sql.OptimizerRuntimeTrackUtil.trackOptimizerRuntime; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; +import static com.facebook.presto.telemetry.TracingManager.setAttributes; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -98,36 +105,68 @@ public Optimizer( this.explain = explain; } + private BaseSpan optimizerSpan(PlanOptimizer optimizer) + { + if (!TracingManager.isRecording()) { + return null; + } + BaseSpan span = TracingManager.getSpan(optimizer.getClass().getSimpleName()); //Recheck if working + + if (optimizer instanceof IterativeOptimizer) { + List ruleNames = ((IterativeOptimizer) optimizer).getRules().stream() + .map(x -> x.getClass().getSimpleName()) + .collect(Collectors.toList()); + + String rulesAsString = String.join(", ", ruleNames); + setAttributes(span, ImmutableMap.of("OPTIMIZER_RULES", rulesAsString)); + } + return scopedSpan(span); + } + public Plan validateAndOptimizePlan(PlanNode root, PlanStage stage) { - planChecker.validateIntermediatePlan(root, session, metadata, warningCollector); + try (BaseSpan ignored = scopedSpan("validate intermediate")) { + planChecker.validateIntermediatePlan(root, session, metadata, warningCollector); + } boolean enableVerboseRuntimeStats = SystemSessionProperties.isVerboseRuntimeStatsEnabled(session); if (stage.ordinal() >= OPTIMIZED.ordinal()) { - for (PlanOptimizer optimizer : planOptimizers) { - if (Thread.currentThread().isInterrupted()) { - throw new PrestoException(QUERY_PLANNING_TIMEOUT, String.format("The query optimizer exceeded the timeout of %s.", getQueryAnalyzerTimeout(session).toString())); - } - long start = System.nanoTime(); - PlanOptimizerResult optimizerResult = optimizer.optimize(root, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector); - requireNonNull(optimizerResult, format("%s returned a null plan", optimizer.getClass().getName())); - if (enableVerboseRuntimeStats || trackOptimizerRuntime(session, optimizer)) { - session.getRuntimeStats().addMetricValue(String.format("optimizer%sTimeNanos", getOptimizerNameForLog(optimizer)), NANO, System.nanoTime() - start); + try (BaseSpan ignored = scopedSpan("validate and optimize")) { + for (PlanOptimizer optimizer : planOptimizers) { + if (Thread.currentThread().isInterrupted()) { + throw new PrestoException(QUERY_PLANNING_TIMEOUT, String.format("The query optimizer exceeded the timeout of %s.", getQueryAnalyzerTimeout(session).toString())); + } + long start = System.nanoTime(); + + PlanOptimizerResult optimizerResult; + + try (BaseSpan opSpanIgnored = !TelemetryConfig.getTracingEnabled() ? null : optimizerSpan(optimizer)) { + optimizerResult = optimizer.optimize(root, session, TypeProvider.viewOf(variableAllocator.getVariables()), variableAllocator, idAllocator, warningCollector); + requireNonNull(optimizerResult, format("%s returned a null plan", optimizer.getClass().getName())); + } + + if (enableVerboseRuntimeStats || trackOptimizerRuntime(session, optimizer)) { + session.getRuntimeStats().addMetricValue(String.format("optimizer%sTimeNanos", getOptimizerNameForLog(optimizer)), NANO, System.nanoTime() - start); + } + TypeProvider types = TypeProvider.viewOf(variableAllocator.getVariables()); + + collectOptimizerInformation(optimizer, root, optimizerResult, types); + root = optimizerResult.getPlanNode(); } - TypeProvider types = TypeProvider.viewOf(variableAllocator.getVariables()); - - collectOptimizerInformation(optimizer, root, optimizerResult, types); - root = optimizerResult.getPlanNode(); } } - if (stage.ordinal() >= OPTIMIZED_AND_VALIDATED.ordinal()) { - // make sure we produce a valid plan after optimizations run. This is mainly to catch programming errors - planChecker.validateFinalPlan(root, session, metadata, warningCollector); + try (BaseSpan ignored = scopedSpan("validate final")) { + if (stage.ordinal() >= OPTIMIZED_AND_VALIDATED.ordinal()) { + // make sure we produce a valid plan after optimizations run. This is mainly to catch programming errors + planChecker.validateFinalPlan(root, session, metadata, warningCollector); + } } TypeProvider types = TypeProvider.viewOf(variableAllocator.getVariables()); - return new Plan(root, types, computeStats(root, types)); + try (BaseSpan ignored = scopedSpan("plan stats")) { + return new Plan(root, types, computeStats(root, types)); + } } private StatsAndCosts computeStats(PlanNode root, TypeProvider types) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java index 711b63bb5a4c9..66dff1e03c860 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.common.analyzer.PreparedQuery; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.analyzer.AnalyzerContext; @@ -24,6 +25,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.LogicalPlanner; import com.google.inject.Inject; @@ -32,6 +34,7 @@ import java.util.concurrent.ExecutorService; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.telemetry.TracingManager.scopedSpan; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -90,8 +93,10 @@ public QueryAnalysis analyze(AnalyzerContext analyzerContext, PreparedQuery prep session.getWarningCollector(), Optional.of(metadataExtractorExecutor)); - Analysis analysis = analyzer.analyzeSemantic(((BuiltInQueryPreparer.BuiltInPreparedQuery) preparedQuery).getStatement(), false); - return new BuiltInQueryAnalysis(analysis); + try (BaseSpan ignored = scopedSpan(TracingEnum.ANALYZE.getName())) { + Analysis analysis = analyzer.analyzeSemantic(((BuiltInQueryPreparer.BuiltInPreparedQuery) preparedQuery).getStatement(), false); + return new BuiltInQueryAnalysis(analysis); + } } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java index 79768c665870a..95032b7b194ca 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java @@ -69,6 +69,7 @@ public class IterativeOptimizer private final List legacyRules; private final RuleIndex ruleIndex; private final Optional logicalPropertiesProvider; + private final Set> rules; public IterativeOptimizer(Metadata metadata, RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, Set> rules) { @@ -98,6 +99,12 @@ public IterativeOptimizer(Metadata metadata, RuleStatsRecorder stats, StatsCalcu this.logicalPropertiesProvider = requireNonNull(logicalPropertiesProvider, "logicalPropertiesProvider is null"); stats.registerAll(newRules); + this.rules = newRules; + } + + public Set> getRules() + { + return rules; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerHandle.java b/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryModule.java similarity index 55% rename from presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerHandle.java rename to presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryModule.java index c700db6ffcecc..05bc45425de80 100644 --- a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryModule.java @@ -11,23 +11,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.tracing; +package com.facebook.presto.telemetry; -import com.facebook.presto.spi.tracing.TracerHandle; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; -public class SimpleTracerHandle - implements TracerHandle -{ - private final String traceToken; - - public SimpleTracerHandle(String traceToken) - { - this.traceToken = traceToken; - } +import static org.weakref.jmx.guice.ExportBinder.newExporter; +/** + * The type Telemetry module. + */ +public class TelemetryModule + implements Module +{ @Override - public String getTraceToken() + public void configure(Binder binder) { - return traceToken; + binder.bind(TracingManager.class).in(Scopes.SINGLETON); + newExporter(binder).export(TracingManager.class).withGeneratedName(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryResource.java b/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryResource.java new file mode 100644 index 0000000000000..0c6ca3b7443b8 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryResource.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.telemetry; + +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.common.TracingConfig; + +import javax.annotation.security.RolesAllowed; +import javax.validation.Valid; +import javax.ws.rs.Consumes; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; + +import static com.facebook.presto.server.security.RoleType.ADMIN; +import static com.facebook.presto.server.security.RoleType.USER; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; + +/** + * The type Telemetry resource. + */ +@Path("/v1/telemetry") +@RolesAllowed({USER, ADMIN}) +public class TelemetryResource +{ + /** + * Update telemetry config properties. + * + * @param tracingConfig the tracing config + * @return the response + * @throws WebApplicationException the web application exception + */ + @PUT + @Path("/config") + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + public Response updateTelemetryConfig( + @Valid TracingConfig tracingConfig) throws WebApplicationException + { + TelemetryConfig.getTelemetryConfig().setTracingEnabled(tracingConfig.isTracingEnabled()); //update tracing property based on json response + return Response.ok("Telemetry config updated").build(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryTracingImpl.java b/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryTracingImpl.java new file mode 100644 index 0000000000000..6e99285860aca --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/telemetry/TelemetryTracingImpl.java @@ -0,0 +1,189 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.telemetry; + +import com.facebook.presto.common.ErrorCode; +import com.facebook.presto.spi.telemetry.BaseSpan; +import com.facebook.presto.spi.telemetry.TelemetryFactory; +import com.facebook.presto.spi.telemetry.TelemetryTracing; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Optional; + +/** + * The type Telemetry tracing. + */ +public class TelemetryTracingImpl + implements TelemetryTracing +{ + /** + * The constant NAME. + */ + public static final String NAME = "otel"; + + private static final TelemetryTracingImpl INSTANCE = new TelemetryTracingImpl(); + + /** + * The type Factory. + */ + public static class Factory + implements TelemetryFactory + { + @Override + public String getName() + { + return NAME; + } + + @Override + public TelemetryTracing create() + { + return INSTANCE; + } + } + + @Override + public void loadConfiguredOpenTelemetry() + { + return; + } + + @Override + public Runnable getCurrentContextWrap(Runnable runnable) + { + return runnable; + } + + @Override + public boolean isRecording() + { + return false; + } + + @Override + public Map getHeadersMap(BaseSpan span) + { + return ImmutableMap.of(); + } + + @Override + public void endSpanOnError(BaseSpan querySpan, Throwable throwable) + { + return; + } + + @Override + public void addEvent(BaseSpan span, String eventName) + { + return; + } + + @Override + public void addEvent(BaseSpan querySpan, String eventName, String eventState) + { + return; + } + + @Override + public void setAttributes(BaseSpan span, Map attributes) + { + return; + } + + @Override + public void recordException(BaseSpan querySpan, String message, RuntimeException runtimeException, ErrorCode errorCode) + { + return; + } + + @Override + public void setSuccess(BaseSpan querySpan) + { + return; + } + + @Override + public BaseSpan getInvalidSpan() + { + return getBaseSpan(); + } + + @Override + public BaseSpan getRootSpan() + { + return getBaseSpan(); + } + + @Override + public BaseSpan getSpan(String spanName) + { + return getBaseSpan(); + } + + @Override + public BaseSpan getSpan(String traceParent, String spanName) + { + return getBaseSpan(); + } + + @Override + public Optional spanString(BaseSpan span) + { + return Optional.empty(); + } + + @Override + public BaseSpan scopedSpan(String name, Boolean... skipSpan) + { + return getBaseSpan(); + } + + @Override + public BaseSpan scopedSpan(BaseSpan span, Boolean... skipSpan) + { + return getBaseSpan(); + } + + @Override + public BaseSpan scopedSpan(BaseSpan parentSpan, String spanName, Boolean... skipSpan) + { + return getBaseSpan(); + } + + @Override + public BaseSpan scopedSpan(String spanName, Map attributes, Boolean... skipSpan) + { + return getBaseSpan(); + } + + @Override + public BaseSpan scopedSpan(BaseSpan parentSpan, String spanName, Map attributes, Boolean... skipSpan) + { + return getBaseSpan(); + } + + @Override + public BaseSpan getSpan(BaseSpan parentSpan, String spanName, Map attributes) + { + return null; + } + + private BaseSpan getBaseSpan() + { + return new BaseSpan() + { + }; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/telemetry/TracingManager.java b/presto-main/src/main/java/com/facebook/presto/telemetry/TracingManager.java new file mode 100644 index 0000000000000..a15db86f289c1 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/telemetry/TracingManager.java @@ -0,0 +1,356 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.telemetry; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.ErrorCode; +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.spi.telemetry.BaseSpan; +import com.facebook.presto.spi.telemetry.TelemetryFactory; +import com.facebook.presto.spi.telemetry.TelemetryTracing; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.collect.Maps.fromProperties; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +/** + * OpenTelemetryManager class creates and manages OpenTelemetry and Tracer instances. + */ +public class TracingManager +{ + private static final Logger log = Logger.get(TracingManager.class); + private static final File OPENTELEMETRY_CONFIGURATION = new File("etc/telemetry-tracing.properties"); + private static final String TRACING_FACTORY_NAME = "tracing-factory.name"; + + private final Map openTelemetryFactories = new ConcurrentHashMap<>(); + private static AtomicReference configuredTelemetryTracing = new AtomicReference<>(new TelemetryTracingImpl()); + + /** + * adds and registers all the OpenTelemetryFactory implementations to support different configurations + * + * @param telemetryFactory the telemetry factory + */ + public void addOpenTelemetryFactory(TelemetryFactory telemetryFactory) + { + requireNonNull(telemetryFactory, "openTelemetryFactory is null"); + log.debug("Adding telemetry factory"); + if (openTelemetryFactories.putIfAbsent(telemetryFactory.getName(), telemetryFactory) != null) { + throw new IllegalArgumentException(format("openTelemetry factory '%s' is already registered", telemetryFactory.getName())); + } + } + + /** + * called from PrestoServer for loading the properties after OpenTelemetryManager is bound and injected + * + * @throws Exception the exception + */ + public void loadConfiguredOpenTelemetry() + throws Exception + { + if (OPENTELEMETRY_CONFIGURATION.exists()) { + Map properties = loadProperties(OPENTELEMETRY_CONFIGURATION); + checkArgument( + !isNullOrEmpty(properties.get(TRACING_FACTORY_NAME)), + "Opentelemetry configuration %s does not contain %s", + OPENTELEMETRY_CONFIGURATION.getAbsoluteFile(), + TRACING_FACTORY_NAME); + + if (properties.isEmpty()) { + log.debug("telemetry properties not loaded"); + } + + properties = new HashMap<>(properties); + String openTelemetryFactoryName = properties.remove(TRACING_FACTORY_NAME); + + checkArgument(!isNullOrEmpty(openTelemetryFactoryName), "otel-factory.name property must be present"); + + TelemetryFactory openTelemetryFactory = openTelemetryFactories.get(openTelemetryFactoryName); + checkState(openTelemetryFactory != null, "Opentelemetry factory %s is not registered", openTelemetryFactoryName); + this.configuredTelemetryTracing.set((TelemetryTracing) openTelemetryFactory.create()); + + log.debug("setting telemetry properties"); + TelemetryConfig.getTelemetryConfig().setTelemetryProperties(properties); + configuredTelemetryTracing.get().loadConfiguredOpenTelemetry(); + } + } + + private static Map loadProperties(File file) + throws IOException + { + Properties properties = new Properties(); + try (InputStream in = Files.newInputStream(file.toPath())) { + properties.load(in); + } + return fromProperties(properties); + } + + /** + * Sets configured telemetry tracing. + * + * @param telemetryTracing the telemetry tracing + */ + public static void setConfiguredTelemetryTracing(TelemetryTracing telemetryTracing) + { + TracingManager.configuredTelemetryTracing.set(telemetryTracing); + } + + /** + * Gets current context wrap. + * + * @param runnable the runnable + * @return the current context wrap + */ + public static Runnable getCurrentContextWrap(Runnable runnable) + { + return configuredTelemetryTracing.get().getCurrentContextWrap(runnable); + } + + /** + * Is recording boolean. + * + * @return the boolean + */ + public static boolean isRecording() + { + return configuredTelemetryTracing.get().isRecording(); + } + + /** + * Gets headers map. + * + * @param span the span + * @return the headers map + */ + public static Map getHeadersMap(BaseSpan span) + { + return configuredTelemetryTracing.get().getHeadersMap(span); + } + + /** + * End span on error. + * + * @param querySpan the query span + * @param throwable the throwable + */ + public static void endSpanOnError(BaseSpan querySpan, Throwable throwable) + { + configuredTelemetryTracing.get().endSpanOnError(querySpan, throwable); + } + + /** + * Add event. + * + * @param querySpan the query span + * @param eventName the event name + */ + public static void addEvent(BaseSpan querySpan, String eventName) + { + configuredTelemetryTracing.get().addEvent(querySpan, eventName); + } + + /** + * Add event. + * + * @param querySpan the query span + * @param eventName the event name + * @param eventState the event state + */ + public static void addEvent(BaseSpan querySpan, String eventName, String eventState) + { + configuredTelemetryTracing.get().addEvent(querySpan, eventName, eventState); + } + + /** + * Sets attributes. + * + * @param span the span + * @param attributes the attributes + */ + public static void setAttributes(BaseSpan span, Map attributes) + { + configuredTelemetryTracing.get().setAttributes(span, attributes); + } + + /** + * Record exception. + * + * @param querySpan the query span + * @param message the message + * @param runtimeException the runtime exception + * @param errorCode the error code + */ + public static void recordException(BaseSpan querySpan, String message, RuntimeException runtimeException, ErrorCode errorCode) + { + configuredTelemetryTracing.get().recordException(querySpan, message, runtimeException, errorCode); + } + + /** + * Sets success. + * + * @param querySpan the query span + */ + public static void setSuccess(BaseSpan querySpan) + { + configuredTelemetryTracing.get().setSuccess(querySpan); + } + + /** + * Gets invalid span. + * + * @return the invalid span + */ +//GetSpans + public static BaseSpan getInvalidSpan() + { + return configuredTelemetryTracing.get().getInvalidSpan(); + } + + /** + * Gets root span. + * + * @return the root span + */ + public static BaseSpan getRootSpan() + { + return configuredTelemetryTracing.get().getRootSpan(); + } + + /** + * Gets span. + * + * @param spanName the span name + * @return the span + */ + public static BaseSpan getSpan(String spanName) + { + return configuredTelemetryTracing.get().getSpan(spanName); + } + + /** + * Gets span. + * + * @param traceParent the trace parent + * @param spanName the span name + * @return the span + */ + public static BaseSpan getSpan(String traceParent, String spanName) + { + return configuredTelemetryTracing.get().getSpan(traceParent, spanName); + } + + /** + * Gets span. + * + * @param parentSpan the parent span + * @param spanName the span name + * @param attributes the attributes + * @return the span + */ + public static BaseSpan getSpan(BaseSpan parentSpan, String spanName, Map attributes) + { + return configuredTelemetryTracing.get().getSpan(parentSpan, spanName, attributes); + } + + /** + * Span string optional. + * + * @param span the span + * @return the optional + */ + public static Optional spanString(BaseSpan span) + { + return configuredTelemetryTracing.get().spanString(span); + } + + //Scoped Span + + /** + * starts a basic span and passes it to overloaded method. This method is used for creating basic spans with no attributes. + * + * @param name name of span to be created + * @param skipSpan optional parameter to implement span sampling by skipping the current span export + * @return base span + */ + public static BaseSpan scopedSpan(String name, Boolean... skipSpan) + { + return configuredTelemetryTracing.get().scopedSpan(name, skipSpan); + } + + /** + * creates a ScopedSpan with the current span. This method is used when we manually create spans in the classes and + * set attributes to them before passing to the Scopedspan. + * + * @param span created span instance + * @param skipSpan optional parameter to implement span sampling by skipping the current span export + * @return base span + */ + public static BaseSpan scopedSpan(BaseSpan span, Boolean... skipSpan) + { + return configuredTelemetryTracing.get().scopedSpan(span, skipSpan); + } + + /** + * Scoped span base span. + * + * @param parentSpan the parent span + * @param spanName the span name + * @param attributes the attributes + * @param skipSpan the skip span + * @return the base span + */ + public static BaseSpan scopedSpan(BaseSpan parentSpan, String spanName, Map attributes, Boolean... skipSpan) + { + return configuredTelemetryTracing.get().scopedSpan(parentSpan, spanName, attributes, skipSpan); + } + + /** + * Scoped span base span. + * + * @param parentSpan the parent span + * @param spanName the span name + * @param skipSpan the skip span + * @return the base span + */ + public static BaseSpan scopedSpan(BaseSpan parentSpan, String spanName, Boolean... skipSpan) + { + return configuredTelemetryTracing.get().scopedSpan(parentSpan, spanName, skipSpan); + } + + /** + * Scoped span base span. + * + * @param spanName the span name + * @param attributes the attributes + * @param skipSpan the skip span + * @return the base span + */ + public static BaseSpan scopedSpan(String spanName, Map attributes, Boolean... skipSpan) + { + return configuredTelemetryTracing.get().scopedSpan(spanName, attributes, skipSpan); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 8d2a9ac22e5bb..fec6548e69512 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -225,9 +225,8 @@ import com.facebook.presto.sql.tree.StartTransaction; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.sql.tree.TruncateTable; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.testing.PageConsumerOperator.PageConsumerOutputFactory; -import com.facebook.presto.tracing.TracerProviderManager; -import com.facebook.presto.tracing.TracingConfig; import com.facebook.presto.transaction.InMemoryTransactionManager; import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.transaction.TransactionManagerConfig; @@ -443,7 +442,6 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, new WarningCollectorConfig(), new NodeSchedulerConfig(), new NodeSpillConfig(), - new TracingConfig(), new CompilerConfig(), new HistoryBasedOptimizationConfig()).getSessionProperties(), new JavaFeaturesConfig(), @@ -543,11 +541,11 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, new ThrowingNodeTtlFetcherManager(), new ThrowingClusterTtlProviderManager(), historyBasedPlanStatisticsManager, - new TracerProviderManager(new TracingConfig()), new NodeStatusNotificationManager(), new ClientRequestFilterManager(), planCheckerProviderManager, - expressionOptimizerManager); + expressionOptimizerManager, + new TracingManager()); connectorManager.addConnectorFactory(globalSystemConnectorFactory); connectorManager.createConnection(GlobalSystemConnector.NAME, GlobalSystemConnector.NAME, ImmutableMap.of()); @@ -558,13 +556,14 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, // rewrite session to use managed SessionPropertyMetadata this.defaultSession = new Session( defaultSession.getQueryId(), + null, + null, withInitialTransaction ? Optional.of(transactionManager.beginTransaction(false)) : defaultSession.getTransactionId(), defaultSession.isClientTransactionSupport(), defaultSession.getIdentity(), defaultSession.getSource(), defaultSession.getCatalog(), defaultSession.getSchema(), - defaultSession.getTraceToken(), defaultSession.getTimeZoneKey(), defaultSession.getLocale(), defaultSession.getRemoteUserAddress(), @@ -579,7 +578,6 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, metadata.getSessionPropertyManager(), defaultSession.getPreparedStatements(), defaultSession.getSessionFunctions(), - defaultSession.getTracer(), defaultSession.getWarningCollector(), defaultSession.getRuntimeStats(), defaultSession.getQueryType()); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java index b2c309addb28b..7a09f8dd4842a 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingConnectorSession.java @@ -53,7 +53,6 @@ public class TestingConnectorSession private final Optional source; private final TimeZoneKey timeZoneKey; private final Locale locale; - private final Optional traceToken; private final long startTime; private final Map> properties; private final Map propertyValues; @@ -65,34 +64,33 @@ public class TestingConnectorSession public TestingConnectorSession(List> properties) { - this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of()); + this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of()); } public TestingConnectorSession(ConnectorIdentity identity, List> properties) { - this(identity.getUser(), identity, Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of()); + this(identity.getUser(), identity, Optional.of("test"), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of()); } public TestingConnectorSession(List> properties, Set clientTags) { - this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), clientTags, Optional.empty(), ImmutableMap.of()); + this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), clientTags, Optional.empty(), ImmutableMap.of()); } public TestingConnectorSession(List> properties, Map propertyValues) { - this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, propertyValues, new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of()); + this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, propertyValues, new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), Optional.empty(), ImmutableMap.of()); } public TestingConnectorSession(List> properties, Optional schema) { - this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), schema, ImmutableMap.of()); + this("user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), UTC_KEY, ENGLISH, System.currentTimeMillis(), properties, ImmutableMap.of(), new FunctionsConfig().isLegacyTimestamp(), Optional.empty(), ImmutableSet.of(), schema, ImmutableMap.of()); } public TestingConnectorSession( String user, ConnectorIdentity identity, Optional source, - Optional traceToken, TimeZoneKey timeZoneKey, Locale locale, long startTime, @@ -107,7 +105,6 @@ public TestingConnectorSession( this.queryId = queryIdGenerator.createNextQueryId().toString(); this.identity = requireNonNull(identity, "identity is null"); this.source = requireNonNull(source, "source is null"); - this.traceToken = requireNonNull(traceToken, "traceToken is null"); this.timeZoneKey = requireNonNull(timeZoneKey, "timeZoneKey is null"); this.locale = requireNonNull(locale, "locale is null"); this.startTime = startTime; @@ -162,12 +159,6 @@ public long getStartTime() return startTime; } - @Override - public Optional getTraceToken() - { - return traceToken; - } - @Override public Optional getClientInfo() { @@ -230,7 +221,6 @@ public String toString() return toStringHelper(this) .add("user", getUser()) .add("source", source.orElse(null)) - .add("traceToken", traceToken.orElse(null)) .add("timeZoneKey", timeZoneKey) .add("locale", locale) .add("startTime", startTime) diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingEventListenerManager.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingEventListenerManager.java index d0bc6e235917e..d9261aad25ccf 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/TestingEventListenerManager.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingEventListenerManager.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.eventlistener.QueryProgressEvent; import com.facebook.presto.spi.eventlistener.QueryUpdatedEvent; import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.google.common.collect.ImmutableMap; import java.util.Optional; @@ -70,7 +71,7 @@ public void publishQueryProgress(QueryProgressEvent queryProgressEvent) } @Override - public void splitCompleted(SplitCompletedEvent splitCompletedEvent) + public void splitCompleted(SplitCompletedEvent splitCompletedEvent, BaseSpan span) { if (configuredEventListener.get().isPresent()) { configuredEventListener.get().get().splitCompleted(splitCompletedEvent); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/TestingTracingManager.java b/presto-main/src/main/java/com/facebook/presto/testing/TestingTracingManager.java new file mode 100644 index 0000000000000..acae83b6dcb61 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/testing/TestingTracingManager.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.testing; + +import com.facebook.presto.spi.telemetry.TelemetryFactory; +import com.facebook.presto.spi.telemetry.TelemetryTracing; +import com.facebook.presto.spi.testing.TestingTelemetryTracing; +import com.facebook.presto.telemetry.TracingManager; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + +public class TestingTracingManager + extends TracingManager +{ + private static final String OTEL_TEST = "oteltest"; + private static final String OTEL = "otel"; + + private final Map openTelemetryFactories = new ConcurrentHashMap<>(); + private static AtomicReference configuredTelemetryTracing = new AtomicReference<>(); + + /** + * adds and registers all the OpenTelemetryFactory implementations to support different configurations + * @param telemetryFactory + */ + public void addOpenTelemetryFactory(TelemetryFactory telemetryFactory) + { + if (OTEL_TEST.equals(telemetryFactory.getName())) { + configuredTelemetryTracing.set((TestingTelemetryTracing) telemetryFactory.create()); + } + + if (OTEL.equals(telemetryFactory.getName())) { + TracingManager.setConfiguredTelemetryTracing((TelemetryTracing) telemetryFactory.create()); + } + } + + public void loadConfiguredOpenTelemetry() + { + configuredTelemetryTracing.get().loadConfiguredOpenTelemetry(); + } + + public boolean isSpansEmpty() + { + return configuredTelemetryTracing.get().isSpansEmpty(); + } + + public boolean spansAnyMatch(String task) + { + return configuredTelemetryTracing.get().spansAnyMatch(task); + } + + public void clearSpanList() + { + configuredTelemetryTracing.get().clearSpanList(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/NoopTracerProvider.java b/presto-main/src/main/java/com/facebook/presto/tracing/NoopTracerProvider.java deleted file mode 100644 index 44bc11d4c2424..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/NoopTracerProvider.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import com.facebook.presto.spi.tracing.NoopTracer; -import com.facebook.presto.spi.tracing.Tracer; -import com.facebook.presto.spi.tracing.TracerHandle; -import com.facebook.presto.spi.tracing.TracerProvider; -import com.google.inject.Inject; - -import java.util.Map; -import java.util.function.Function; - -public class NoopTracerProvider - implements TracerProvider -{ - public static final NoopTracerProvider NOOP_TRACER_PROVIDER = new NoopTracerProvider(); - public static final NoopTracer NOOP_TRACER = new NoopTracer(); - public static final TracerHandle NOOP_TRACER_HANDLE = new NoopTracerHandle(); - public static final Function, TracerHandle> NOOP_HANDLE_GENERATOR = headers -> NOOP_TRACER_HANDLE; - - @Inject - public NoopTracerProvider() {} - - @Override - public String getName() - { - return "NOOP tracer provider"; - } - - @Override - public String getTracerType() - { - return TracingConfig.TracerType.NOOP; - } - - @Override - public Function, TracerHandle> getHandleGenerator() - { - return NOOP_HANDLE_GENERATOR; - } - - @Override - public Tracer getNewTracer(TracerHandle handle) - { - return NOOP_TRACER; - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/QueryStateTracingListener.java b/presto-main/src/main/java/com/facebook/presto/tracing/QueryStateTracingListener.java deleted file mode 100644 index 8a9829d9f369b..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/QueryStateTracingListener.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import com.facebook.presto.execution.QueryState; -import com.facebook.presto.execution.StateMachine; -import com.facebook.presto.spi.tracing.Tracer; - -import static java.util.Objects.requireNonNull; - -public class QueryStateTracingListener - implements StateMachine.StateChangeListener -{ - private QueryState previousState; - private final Tracer tracer; - - public QueryStateTracingListener(Tracer tracer) - { - this.tracer = requireNonNull(tracer, "tracer is null"); - } - - @Override - public synchronized void stateChanged(QueryState newState) - { - if (previousState == null) { - // Initial state condition - tracer.startBlock(newState.toString(), ""); - previousState = newState; - return; - } - - tracer.endBlock(previousState.toString(), ""); - if (newState.isDone()) { - // Last state - previousState = newState; - tracer.endTrace("Query finished with state " + newState); - return; - } - - tracer.startBlock(newState.toString(), ""); - previousState = newState; - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracer.java b/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracer.java deleted file mode 100644 index fb20bbe0fc993..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracer.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.tracing.Tracer; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; - -import static com.facebook.presto.spi.StandardErrorCode.DISTRIBUTED_TRACING_ERROR; -import static com.google.common.base.MoreObjects.toStringHelper; - -public class SimpleTracer - implements Tracer -{ - public final Map blockMap = new ConcurrentHashMap<>(); - public final Map recorderBlockMap = new LinkedHashMap<>(); - public final List pointList = new CopyOnWriteArrayList<>(); - public final String traceToken; - - public SimpleTracer(String traceToken) - { - this.traceToken = traceToken; - addPoint("Start tracing"); - } - - @Override - public void addPoint(String annotation) - { - pointList.add(new SimpleTracerPoint(annotation)); - } - - @Override - public void startBlock(String blockName, String annotation) - { - if (blockMap.containsKey(blockName)) { - throw new PrestoException(DISTRIBUTED_TRACING_ERROR, "Duplicated block inserted: " + blockName); - } - SimpleTracerBlock block = new SimpleTracerBlock(annotation); - blockMap.put(blockName, block); - synchronized (recorderBlockMap) { - recorderBlockMap.put(blockName, block); - } - } - - @Override - public void addPointToBlock(String blockName, String annotation) - { - if (!blockMap.containsKey(blockName)) { - throw new PrestoException(DISTRIBUTED_TRACING_ERROR, "Adding point to non-existing block: " + blockName); - } - blockMap.get(blockName).addPoint(new SimpleTracerPoint(annotation)); - } - - @Override - public void endBlock(String blockName, String annotation) - { - if (!blockMap.containsKey(blockName)) { - throw new PrestoException(DISTRIBUTED_TRACING_ERROR, "Trying to end a non-existing block: " + blockName); - } - blockMap.remove(blockName); - synchronized (recorderBlockMap) { - recorderBlockMap.get(blockName).end(annotation); - } - } - - @Override - public void endTrace(String annotation) - { - pointList.add(new SimpleTracerPoint(annotation)); - } - - @Override - public String getTracerId() - { - if (traceToken != null) { - return traceToken; - } - return tracerName; - } - - public String toString() - { - return toStringHelper(this) - .add("points", pointList) - .add("blocks", recorderBlockMap) - .toString(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerBlock.java b/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerBlock.java deleted file mode 100644 index 5cdd88a2f868b..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerBlock.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -import static com.google.common.base.MoreObjects.toStringHelper; - -public class SimpleTracerBlock -{ - private final List points = new CopyOnWriteArrayList<>(); - private final long startTime = System.nanoTime(); - private long endTime; - - public SimpleTracerBlock(String annotation) - { - addPoint(new SimpleTracerPoint(annotation)); - } - - public long getStartTime() - { - return startTime; - } - - public void end(String annotation) - { - endTime = System.nanoTime(); - if (startTime > endTime) { - throw new RuntimeException("Block start time " + startTime + " is greater than end time " + endTime); - } - addPoint(new SimpleTracerPoint(annotation)); - } - - public long getEndTime() - { - return endTime; - } - - public void addPoint(SimpleTracerPoint point) - { - points.add(point); - } - - public String toString() - { - return toStringHelper(this) - .add("start time", startTime) - .add("end time", endTime) - .add("points", points) - .toString(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerPoint.java b/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerPoint.java deleted file mode 100644 index 8cff367ba6aee..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerPoint.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -public class SimpleTracerPoint -{ - private final String annotation; - private final long time = System.nanoTime(); - - public SimpleTracerPoint(String annotation) - { - this.annotation = requireNonNull(annotation, "annotation is null"); - } - - public String getAnnotation() - { - return annotation; - } - - public long getTime() - { - return time; - } - - public String toString() - { - return toStringHelper(this) - .add("time", time) - .add("annotation", annotation) - .toString(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerProvider.java b/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerProvider.java deleted file mode 100644 index 3fe11a5661e0b..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/SimpleTracerProvider.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import com.facebook.presto.spi.tracing.Tracer; -import com.facebook.presto.spi.tracing.TracerHandle; -import com.facebook.presto.spi.tracing.TracerProvider; -import com.google.inject.Inject; - -import java.util.Map; -import java.util.function.Function; - -import static com.facebook.presto.client.PrestoHeaders.PRESTO_TRACE_TOKEN; - -public class SimpleTracerProvider - implements TracerProvider -{ - @Inject - public SimpleTracerProvider() - { - } - - @Override - public String getName() - { - return "Simple tracer provider"; - } - - @Override - public String getTracerType() - { - return TracingConfig.TracerType.SIMPLE; - } - - @Override - public Function, TracerHandle> getHandleGenerator() - { - return headers -> new SimpleTracerHandle(headers.get(PRESTO_TRACE_TOKEN)); - } - - @Override - public Tracer getNewTracer(TracerHandle handle) - { - SimpleTracerHandle tracerHandle = (SimpleTracerHandle) handle; - return new SimpleTracer(tracerHandle.getTraceToken()); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/TracerProviderManager.java b/presto-main/src/main/java/com/facebook/presto/tracing/TracerProviderManager.java deleted file mode 100644 index 6c1f35cd061fa..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/TracerProviderManager.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import com.facebook.presto.spi.tracing.TracerProvider; -import com.google.inject.Inject; - -import javax.annotation.Nullable; - -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; - -public class TracerProviderManager -{ - private final String tracerType; - private final boolean systemTracingEnabled; - @Nullable - private TracerProvider tracerProvider; - - @Inject - public TracerProviderManager(TracingConfig config) - { - requireNonNull(config, "config is null"); - - this.tracerType = config.getTracerType(); - boolean enableDistributedTracing = config.getEnableDistributedTracing(); - TracingConfig.DistributedTracingMode distributedTracingMode = config.getDistributedTracingMode(); - this.systemTracingEnabled = enableDistributedTracing && distributedTracingMode.name().equalsIgnoreCase(TracingConfig.DistributedTracingMode.ALWAYS_TRACE.name()); - } - - public void addTracerProviderFactory(TracerProvider provider) - { - if (!tracerType.equals(provider.getTracerType())) { - throw new IllegalArgumentException( - format( - "Plugin-configured tracer provider ('%s') does not match system-configured provider ('%s').", - provider.getName(), - tracerType)); - } - if (systemTracingEnabled) { - if (tracerProvider != null) { - throw new IllegalArgumentException(format("Only a single plugin should set the tracer provider ('%s').", tracerProvider.getTracerType())); - } - tracerProvider = provider; - } - } - - public void loadTracerProvider() - { - if (tracerProvider != null) { - return; - } - // open-telemetry plugin not used / tracer provider not specified or not matching system config / tracing disabled - // Check if SimpleTracer is configured and tracing is enabled - // Otherwise, use Noop implementation - if (tracerType.equals(TracingConfig.TracerType.SIMPLE) && systemTracingEnabled) { - tracerProvider = new SimpleTracerProvider(); - } - else { - tracerProvider = new NoopTracerProvider(); - } - } - - public TracerProvider getTracerProvider() - { - if (tracerProvider != null) { - return tracerProvider; - } - return new NoopTracerProvider(); - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/TracingConfig.java b/presto-main/src/main/java/com/facebook/presto/tracing/TracingConfig.java deleted file mode 100644 index e92d658930490..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/tracing/TracingConfig.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import com.facebook.airlift.configuration.Config; -import com.facebook.presto.spi.PrestoException; - -import static com.facebook.presto.spi.StandardErrorCode.DISTRIBUTED_TRACING_ERROR; - -public class TracingConfig -{ - public static class TracerType - { - public static final String NOOP = "noop"; - public static final String SIMPLE = "simple"; - public static final String OTEL = "otel"; - } - - public enum DistributedTracingMode - { - NO_TRACE, - ALWAYS_TRACE, - SAMPLE_BASED - } - - private String tracerType = TracerType.NOOP; - private DistributedTracingMode tracingMode = DistributedTracingMode.NO_TRACE; - private boolean enableDistributedTracing; - - public String getTracerType() - { - return this.tracerType; - } - - @Config("tracing.tracer-type") - public TracingConfig setTracerType(String tracerType) - { - this.tracerType = tracerType; - return this; - } - - public boolean getEnableDistributedTracing() - { - return enableDistributedTracing; - } - - @Config("tracing.enable-distributed-tracing") - public TracingConfig setEnableDistributedTracing(boolean enableDistributedTracing) - { - this.enableDistributedTracing = enableDistributedTracing; - return this; - } - - public DistributedTracingMode getDistributedTracingMode() - { - return tracingMode; - } - - @Config("tracing.distributed-tracing-mode") - public TracingConfig setDistributedTracingMode(String tracingMode) - { - if (tracingMode.equalsIgnoreCase(DistributedTracingMode.SAMPLE_BASED.name())) { - throw new PrestoException( - DISTRIBUTED_TRACING_ERROR, - "SAMPLE_BASED Tracing Mode is currently not supported."); - } - this.tracingMode = DistributedTracingMode.valueOf(tracingMode.toUpperCase()); - return this; - } -} diff --git a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java index 2ae96279b15b7..2d513059ecdd0 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/MockRemoteTaskFactory.java @@ -43,6 +43,7 @@ import com.facebook.presto.spi.plan.StageExecutionDescriptor; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.spiller.SpillSpaceTracker; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.planner.PlanFragment; @@ -148,7 +149,7 @@ public MockRemoteTask createTableScanTask(TaskId taskId, InternalNode newNode, L nodeStatsTracker, true, new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()), - SchedulerStatsTracker.NOOP); + SchedulerStatsTracker.NOOP, null); } @Override @@ -162,7 +163,8 @@ public MockRemoteTask createRemoteTask( NodeTaskMap.NodeStatsTracker nodeStatsTracker, boolean summarizeTaskInfo, TableWriteInfo tableWriteInfo, - SchedulerStatsTracker schedulerStatsTracker) + SchedulerStatsTracker schedulerStatsTracker, + BaseSpan stageSpan) { return new MockRemoteTask(taskId, fragment, node.getNodeIdentifier(), executor, scheduledExecutor, initialSplits, nodeStatsTracker); } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java b/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java index cdee04d9bdc83..8e07ecdbe46b2 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TaskTestUtils.java @@ -66,6 +66,7 @@ import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.testing.TestingMetadata.TestingColumnHandle; import com.facebook.presto.testing.TestingMetadata.TestingTableHandle; import com.facebook.presto.testing.TestingSplit; @@ -195,7 +196,7 @@ public static LocalExecutionPlanner createTestingPlanner() public static TaskInfo updateTask(SqlTask sqlTask, List taskSources, OutputBuffers outputBuffers) { - return sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), taskSources, outputBuffers, Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + return sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), taskSources, outputBuffers, Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), TracingManager.getRootSpan()); } public static SplitMonitor createTestSplitMonitor() diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java index b17cf1548e0db..e5f0b5318ec32 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestMemoryRevokingScheduler.java @@ -38,6 +38,7 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.planner.LocalExecutionPlanner; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.base.Functions; import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; @@ -703,7 +704,8 @@ private TestOperatorContext createTestingOperatorContexts(SqlTask sqlTask, Strin Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), false)), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), + TracingManager.getRootSpan()); // use implicitly created task context from updateTask. It should be the only task in this QueryContext's tasks TaskContext taskContext = sqlTask.getQueryContext().getTaskContextByTaskId(sqlTask.getTaskId()); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java index 00c8d0afd4d7c..7aaff33320581 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlStageExecution.java @@ -109,7 +109,8 @@ private void testFinalStageInfoInternal() executor, new NoOpFailureDetector(), new SplitSchedulerStats(), - new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())); + new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()), + null); stage.setOutputBuffers(createInitialEmptyOutputBuffers(ARBITRARY)); // add listener that fetches stage info when the final status is available diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java index d9fea75b9b0c6..cb738c82b7188 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTask.java @@ -35,6 +35,8 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.planner.LocalExecutionPlanner; +import com.facebook.presto.telemetry.TracingManager; +import com.facebook.presto.testing.TestingTracingManager; import com.google.common.base.Functions; import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; @@ -84,6 +86,7 @@ public class TestSqlTask private final ScheduledExecutorService taskNotificationExecutor; private final ScheduledExecutorService driverYieldExecutor; private final SqlTaskExecutionFactory sqlTaskExecutionFactory; + private TestingTracingManager testingTracingManager; private final AtomicInteger nextTaskId = new AtomicInteger(); @@ -125,7 +128,7 @@ public void testEmptyQuery() ImmutableList.of(), createInitialEmptyOutputBuffers(PARTITIONED) .withNoMoreBufferIds(), - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), TracingManager.getRootSpan()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); @@ -136,7 +139,7 @@ public void testEmptyQuery() ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(), true)), createInitialEmptyOutputBuffers(PARTITIONED) .withNoMoreBufferIds(), - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), TracingManager.getRootSpan()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED); taskInfo = sqlTask.getTaskInfo(); @@ -153,7 +156,7 @@ public void testSimpleQuery() Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), TracingManager.getRootSpan()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); @@ -203,7 +206,7 @@ public void testCancel() createInitialEmptyOutputBuffers(PARTITIONED) .withBuffer(OUT, 0) .withNoMoreBufferIds(), - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), TracingManager.getRootSpan()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); @@ -230,7 +233,7 @@ public void testAbort() Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), TracingManager.getRootSpan()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java index 78e3b957de318..e8004afbf87fe 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskExecution.java @@ -188,7 +188,8 @@ public void testSimple(PipelineExecutionStrategy executionStrategy) localExecutionPlan, taskExecutor, taskNotificationExecutor, - createTestSplitMonitor()); + createTestSplitMonitor(), + null); // // test body @@ -450,7 +451,8 @@ public void testComplex(PipelineExecutionStrategy executionStrategy) localExecutionPlan, taskExecutor, taskNotificationExecutor, - createTestSplitMonitor()); + createTestSplitMonitor(), + null); // // test body diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java index 15496a48bef0d..761634dab09f2 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestSqlTaskManager.java @@ -39,6 +39,7 @@ import com.facebook.presto.spiller.NodeSpillConfig; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.OrderingCompiler; +import com.facebook.presto.telemetry.TracingManager; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; @@ -351,7 +352,8 @@ private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, Immuta Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, splits, true)), outputBuffers, - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), + TracingManager.getRootSpan()); } private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, OutputBuffers outputBuffers) @@ -372,7 +374,8 @@ private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, Output Optional.of(PLAN_FRAGMENT), ImmutableList.of(), outputBuffers, - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), + TracingManager.getRootSpan()); } public static class MockExchangeClientSupplier diff --git a/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStateMachine.java b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStateMachine.java index 5fded52c9f5f7..6ca73479f6d30 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStateMachine.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/TestStageExecutionStateMachine.java @@ -289,6 +289,6 @@ private static void assertState(StageExecutionStateMachine stateMachine, StageEx private StageExecutionStateMachine createStageStateMachine() { - return new StageExecutionStateMachine(STAGE_ID, executor, new SplitSchedulerStats(), false); + return new StageExecutionStateMachine(STAGE_ID, executor, new SplitSchedulerStats(), false, null); } } diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java index 34a707792fccd..8d22f66dcd76a 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestAdaptivePhasedExecutionPolicy.java @@ -48,7 +48,6 @@ import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.testing.TestingMetadata; import com.facebook.presto.testing.TestingTransactionHandle; -import com.facebook.presto.tracing.TracingConfig; import com.facebook.presto.util.FinalizerService; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -104,7 +103,6 @@ public void testCreateExecutionSchedule() new WarningCollectorConfig(), new NodeSchedulerConfig(), new NodeSpillConfig(), - new TracingConfig(), new CompilerConfig(), new HistoryBasedOptimizationConfig()))).build(); AdaptivePhasedExecutionPolicy policy = new AdaptivePhasedExecutionPolicy(); @@ -143,7 +141,8 @@ private StageExecutionAndScheduler getStageExecutionAndScheduler(int stage, Plan newDirectExecutorService(), new NoOpFailureDetector(), new SplitSchedulerStats(), - new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())); + new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()), + null); StageLinkage stageLinkage = new StageLinkage(fragmentId, (id, tasks, noMoreExchangeLocations) -> {}, ImmutableSet.of()); StageScheduler stageScheduler = new FixedCountScheduler(stageExecution, ImmutableList.of()); StageExecutionAndScheduler scheduler = new StageExecutionAndScheduler(stageExecution, stageLinkage, stageScheduler); diff --git a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java index 00c4483a755c5..4db75f063db34 100644 --- a/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java +++ b/presto-main/src/test/java/com/facebook/presto/execution/scheduler/TestSourcePartitionedScheduler.java @@ -546,7 +546,8 @@ private SqlStageExecution createSqlStageExecution(SubPlan tableScanPlan, NodeTas queryExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), - new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())); + new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()), + null); stage.setOutputBuffers(createInitialEmptyOutputBuffers(PARTITIONED) .withBuffer(OUT, 0) diff --git a/presto-main/src/test/java/com/facebook/presto/memory/TestHighMemoryTaskKiller.java b/presto-main/src/test/java/com/facebook/presto/memory/TestHighMemoryTaskKiller.java index 7729f01d371ca..109886fb1817b 100644 --- a/presto-main/src/test/java/com/facebook/presto/memory/TestHighMemoryTaskKiller.java +++ b/presto-main/src/test/java/com/facebook/presto/memory/TestHighMemoryTaskKiller.java @@ -36,6 +36,7 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.gen.OrderingCompiler; import com.facebook.presto.sql.planner.LocalExecutionPlanner; +import com.facebook.presto.telemetry.TracingManager; import com.google.common.base.Functions; import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; @@ -128,7 +129,8 @@ public void updateTaskMemory(SqlTask sqlTask, long systemMemory) ImmutableList.of(), createInitialEmptyOutputBuffers(PARTITIONED) .withNoMoreBufferIds(), - Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()))); + Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())), + TracingManager.getRootSpan()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); TaskContext taskContext = sqlTask.getTaskContext().get(); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java index 53186af9f835e..ee67bbf7754c3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java @@ -167,7 +167,6 @@ private void assertCurrentDateAtInstant(TimeZoneKey timeZoneKey, long instant) "test", new ConnectorIdentity("test", Optional.empty(), Optional.empty()), Optional.empty(), - Optional.empty(), timeZoneKey, US, instant, diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java b/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java index d4eb03bb64a88..5be4fabaf2980 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestQuerySessionSupplier.java @@ -104,7 +104,7 @@ public WarningCollector create(WarningHandlingLevel warningHandlingLevel) return WarningCollector.NOOP; } }; - Session session = sessionSupplier.createSession(new QueryId("test_query_id"), context, warningCollectorFactory); + Session session = sessionSupplier.createSession(new QueryId("test_query_id"), null, null, context, warningCollectorFactory); assertEquals(session.getQueryId(), new QueryId("test_query_id")); assertEquals(session.getUser(), "testUser"); @@ -178,6 +178,6 @@ public WarningCollector create(WarningHandlingLevel warningHandlingLevel) return WarningCollector.NOOP; } }; - sessionSupplier.createSession(new QueryId("test_query_id"), context, warningCollectorFactory); + sessionSupplier.createSession(new QueryId("test_query_id"), null, null, context, warningCollectorFactory); } } diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestThriftServerInfoIntegration.java b/presto-main/src/test/java/com/facebook/presto/server/TestThriftServerInfoIntegration.java index e1e713ebf711c..178961249dc35 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestThriftServerInfoIntegration.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestThriftServerInfoIntegration.java @@ -45,6 +45,7 @@ import com.facebook.presto.server.thrift.ThriftServerInfoClient; import com.facebook.presto.server.thrift.ThriftServerInfoService; import com.facebook.presto.spi.NodeState; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -194,7 +195,7 @@ public void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments } @Override - public TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, Optional tableWriteInfo) + public TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, Optional tableWriteInfo, BaseSpan span) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/test/java/com/facebook/presto/server/TestThriftTaskIntegration.java b/presto-main/src/test/java/com/facebook/presto/server/TestThriftTaskIntegration.java index 81d4ab3b650a2..7eca12f211fb2 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/TestThriftTaskIntegration.java +++ b/presto-main/src/test/java/com/facebook/presto/server/TestThriftTaskIntegration.java @@ -43,6 +43,7 @@ import com.facebook.presto.metadata.MetadataUpdates; import com.facebook.presto.server.thrift.ThriftTaskClient; import com.facebook.presto.server.thrift.ThriftTaskService; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; @@ -223,7 +224,7 @@ public void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments } @Override - public TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, Optional tableWriteInfo) + public TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, Optional tableWriteInfo, BaseSpan span) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java index 6791964a215dd..a1ed15ea48e10 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java @@ -338,7 +338,7 @@ private RemoteTask createRemoteTask(HttpRemoteTaskFactory httpRemoteTaskFactory) new NodeTaskMap.NodeStatsTracker(i -> {}, i -> {}, (age, i) -> {}), true, new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()), - SchedulerStatsTracker.NOOP); + SchedulerStatsTracker.NOOP, null); } private static HttpRemoteTaskFactory createHttpRemoteTaskFactory(TestingTaskResource testingTaskResource, boolean useThriftEncoding) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java index 83616f63c7779..523cd7acc26d9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java @@ -27,7 +27,6 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spiller.NodeSpillConfig; import com.facebook.presto.sql.planner.CompilerConfig; -import com.facebook.presto.tracing.TracingConfig; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; @@ -202,7 +201,6 @@ public void testWindowOrderByAnalysis() new WarningCollectorConfig(), new NodeSchedulerConfig(), new NodeSpillConfig(), - new TracingConfig(), new CompilerConfig(), new HistoryBasedOptimizationConfig()))).build(); assertFails(session, WINDOW_FUNCTION_ORDERBY_LITERAL, @@ -598,7 +596,6 @@ public void testTooManyGroupingElements() new WarningCollectorConfig(), new NodeSchedulerConfig(), new NodeSpillConfig(), - new TracingConfig(), new CompilerConfig(), new HistoryBasedOptimizationConfig()))).build(); analyze(session, "SELECT a, b, c, d, e, f, g, h, i, j, k, SUM(l)" + diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java index caa37100245c7..7e6ffbfbfb7eb 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/sanity/TestValidateStreamingJoins.java @@ -42,7 +42,6 @@ import com.facebook.presto.tpch.TpchColumnHandle; import com.facebook.presto.tpch.TpchTableHandle; import com.facebook.presto.tpch.TpchTableLayoutHandle; -import com.facebook.presto.tracing.TracingConfig; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.AfterClass; @@ -87,7 +86,6 @@ public void setup() new WarningCollectorConfig(), new NodeSchedulerConfig(), new NodeSpillConfig(), - new TracingConfig(), new CompilerConfig(), new HistoryBasedOptimizationConfig()))) .setCatalog("local") diff --git a/presto-main/src/test/java/com/facebook/presto/telemetry/TestTelemetryResource.java b/presto-main/src/test/java/com/facebook/presto/telemetry/TestTelemetryResource.java new file mode 100644 index 0000000000000..e0b5ada8474b5 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/telemetry/TestTelemetryResource.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.telemetry; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.jetty.JettyHttpClient; +import com.facebook.presto.common.TracingConfig; +import com.facebook.presto.server.testing.TestingPrestoServer; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.airlift.http.client.JsonBodyGenerator.jsonBodyGenerator; +import static com.facebook.airlift.http.client.Request.Builder.preparePut; +import static com.facebook.airlift.http.client.StringResponseHandler.createStringResponseHandler; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.airlift.testing.Closeables.closeQuietly; +import static com.facebook.presto.server.RequestHelpers.setContentTypeHeaders; +import static org.testng.Assert.assertEquals; + +public class TestTelemetryResource +{ + private HttpClient client; + private TestingPrestoServer server; + + @BeforeClass + public void setup() + throws Exception + { + client = new JettyHttpClient(); + server = new TestingPrestoServer(); + } + + @AfterClass(alwaysRun = true) + public void teardown() + { + closeQuietly(server); + closeQuietly(client); + server = null; + client = null; + } + + @Test + public void testUpdateTelemetryConfig() + { + Request.Builder requestBuilder = setContentTypeHeaders(false, preparePut()); + Request request = requestBuilder.setUri(server.getBaseUrl().resolve("/v1/telemetry/config")) + .setBodyGenerator(jsonBodyGenerator(jsonCodec(TracingConfig.class), new TracingConfig(true))) + .build(); + assertEquals(client.execute(request, createStringResponseHandler()).getStatusCode(), 200); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/tracing/TestTracingConfig.java b/presto-main/src/test/java/com/facebook/presto/tracing/TestTracingConfig.java deleted file mode 100644 index 13a58c276c25e..0000000000000 --- a/presto-main/src/test/java/com/facebook/presto/tracing/TestTracingConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing; - -import com.facebook.airlift.configuration.testing.ConfigAssertions; -import com.google.common.collect.ImmutableMap; -import org.testng.annotations.Test; - -import java.util.Map; - -import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; - -public class TestTracingConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(ConfigAssertions.recordDefaults(TracingConfig.class) - .setTracerType("noop") - .setEnableDistributedTracing(false) - .setDistributedTracingMode(TracingConfig.DistributedTracingMode.NO_TRACE.name())); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = new ImmutableMap.Builder() - .put("tracing.tracer-type", "simple") - .put("tracing.enable-distributed-tracing", "true") - .put("tracing.distributed-tracing-mode", "always_trace") - .build(); - - TracingConfig expected = new TracingConfig() - .setTracerType("simple") - .setEnableDistributedTracing(true) - .setDistributedTracingMode(TracingConfig.DistributedTracingMode.ALWAYS_TRACE.name()); - assertFullMapping(properties, expected); - } -} diff --git a/presto-main/src/test/java/com/facebook/presto/tracing/testing/TestSimpleTracer.java b/presto-main/src/test/java/com/facebook/presto/tracing/testing/TestSimpleTracer.java deleted file mode 100644 index 810903c26df60..0000000000000 --- a/presto-main/src/test/java/com/facebook/presto/tracing/testing/TestSimpleTracer.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.tracing.testing; - -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.tracing.TracerHandle; -import com.facebook.presto.tracing.SimpleTracer; -import com.facebook.presto.tracing.SimpleTracerProvider; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; - -import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; -import static com.facebook.presto.spi.StandardErrorCode.DISTRIBUTED_TRACING_ERROR; -import static java.util.concurrent.Executors.newFixedThreadPool; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.expectThrows; - -public class TestSimpleTracer -{ - private final SimpleTracerProvider tracerProvider = new SimpleTracerProvider(); - private final ExecutorService executor = newFixedThreadPool(16, daemonThreadsNamed("presto-testing-tracer-threadpool")); - private final Random random = new Random(); - private final int numThreads = 10; - - public TestSimpleTracer() - { - } - - @Test - public void testAddPoint() - { - Map testHeaders = new HashMap<>(); - TracerHandle testTracerHandle = tracerProvider.getHandleGenerator().apply(testHeaders); - SimpleTracer tracer = (SimpleTracer) tracerProvider.getNewTracer(testTracerHandle); - List> futures = new ArrayList<>(); - for (int i = 0; i < numThreads; i++) { - CompletableFuture future = new CompletableFuture<>(); - executor.submit(() -> { - for (int j = 0; j < 20; j++) { - tracer.addPoint("test-point"); - try { - Thread.sleep(random.nextInt(5)); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - } - future.complete(null); - }); - futures.add(future); - } - CompletableFuture.allOf(futures.toArray(new CompletableFuture[numThreads])).thenApply(v -> { - tracer.endTrace("trace ended"); - assertEquals(tracer.pointList.size(), 202); - return null; - }).join(); - } - - @Test - public void testAddBlock() - { - Map testHeaders = new HashMap<>(); - TracerHandle testTracerHandle = tracerProvider.getHandleGenerator().apply(testHeaders); - SimpleTracer tracer = (SimpleTracer) tracerProvider.getNewTracer(testTracerHandle); - List> futures = new ArrayList<>(); - for (int i = 0; i < numThreads; i++) { - CompletableFuture future = new CompletableFuture<>(); - int threadNum = i; - executor.submit(() -> { - for (int j = 0; j < 20; j++) { - tracer.startBlock("test-block-" + threadNum + "." + j, ""); - tracer.addPointToBlock("test-block-" + threadNum + "." + j, "point added"); - try { - Thread.sleep(random.nextInt(20)); - } - catch (InterruptedException e) { - e.printStackTrace(); - } - tracer.endBlock("test-block-" + threadNum + "." + j, ""); - } - future.complete(null); - }); - futures.add(future); - } - CompletableFuture.allOf(futures.toArray(new CompletableFuture[numThreads])).thenApply(v -> { - tracer.endTrace("trace ended"); - assertEquals(tracer.recorderBlockMap.size(), 200); - return null; - }).join(); - } - - @Test - public void testBlockErrors() - { - Map testHeaders = new HashMap<>(); - TracerHandle testTracerHandle = tracerProvider.getHandleGenerator().apply(testHeaders); - SimpleTracer tracer = (SimpleTracer) tracerProvider.getNewTracer(testTracerHandle); - - // Duplicate block - PrestoException exception = expectThrows(PrestoException.class, () -> { - tracer.startBlock("test-block", ""); - tracer.startBlock("test-block", ""); - }); - assertEquals(exception.getErrorCode(), DISTRIBUTED_TRACING_ERROR.toErrorCode()); - tracer.endBlock("test-block", ""); - - // Deleting non-existing block - exception = expectThrows(PrestoException.class, () -> { - tracer.startBlock("test-block", ""); - tracer.endBlock("test-block-non-existing", ""); - }); - assertEquals(exception.getErrorCode(), DISTRIBUTED_TRACING_ERROR.toErrorCode()); - tracer.endBlock("test-block", ""); - - // Adding point to non-existing block - exception = expectThrows(PrestoException.class, () -> tracer.addPointToBlock("test-block", "Adding point to non-existing block")); - assertEquals(exception.getErrorCode(), DISTRIBUTED_TRACING_ERROR.toErrorCode()); - } -} diff --git a/presto-native-sidecar-plugin/pom.xml b/presto-native-sidecar-plugin/pom.xml index 6b8f1badd289d..0466570fb39d3 100644 --- a/presto-native-sidecar-plugin/pom.xml +++ b/presto-native-sidecar-plugin/pom.xml @@ -62,6 +62,12 @@ com.squareup.okhttp3 okhttp + + + org.jetbrains.kotlin + kotlin-stdlib + + diff --git a/presto-open-telemetry/pom.xml b/presto-open-telemetry/pom.xml index 5104cdc983528..c73bcbcf6c565 100644 --- a/presto-open-telemetry/pom.xml +++ b/presto-open-telemetry/pom.xml @@ -17,34 +17,42 @@ - - com.google.inject - guice + + com.facebook.airlift + log com.google.guava guava + + + io.opentelemetry.instrumentation + opentelemetry-okhttp-3.0 + 2.4.0-alpha + runtime + - io.opentelemetry - opentelemetry-api + com.squareup.okhttp3 + okhttp + runtime io.opentelemetry - opentelemetry-context + opentelemetry-api io.opentelemetry - opentelemetry-exporter-otlp + opentelemetry-context io.opentelemetry - opentelemetry-extension-trace-propagators + opentelemetry-exporter-otlp @@ -64,7 +72,7 @@ io.opentelemetry - opentelemetry-semconv + opentelemetry-sdk-testing @@ -109,5 +117,32 @@ jol-core provided + + + com.google.errorprone + error_prone_annotations + + + + org.testng + testng + test + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + io.opentelemetry:opentelemetry-api:jar:1.38.0 + + + + + + diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryBuilder.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryBuilder.java deleted file mode 100644 index e50d52f2b6918..0000000000000 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryBuilder.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.opentelemetry; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.context.propagation.TextMapPropagator; -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; -import io.opentelemetry.extension.trace.propagation.B3Propagator; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; - -public final class OpenTelemetryBuilder -{ - private OpenTelemetryBuilder() - { - throw new UnsupportedOperationException("This is a utility class and cannot be instantiated."); - } - - /** - * Get instance of propagator. - * Currently, only B3_SINGLE_HEADER can be passed in. - */ - private static TextMapPropagator getPropagatorInstance(String contextPropagator) - { - TextMapPropagator propagator; - if (contextPropagator.equals(OpenTelemetryContextPropagator.W3C)) { - propagator = W3CTraceContextPropagator.getInstance(); - } - else if (contextPropagator.equals(OpenTelemetryContextPropagator.B3_SINGLE_HEADER)) { - propagator = B3Propagator.injectingSingleHeader(); - } - else { - propagator = B3Propagator.injectingMultiHeaders(); - } - return propagator; - } - - public static OpenTelemetry build(String contextPropagator) - { - Resource resource = Resource.getDefault() - .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "presto"))); - - SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(BatchSpanProcessor.builder(OtlpGrpcSpanExporter.builder().setEndpoint(System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")).build()).build()) - .setResource(resource) - .build(); - - return OpenTelemetrySdk.builder() - .setTracerProvider(sdkTracerProvider) - .setPropagators(ContextPropagators.create(OpenTelemetryBuilder.getPropagatorInstance(contextPropagator))) - .buildAndRegisterGlobal(); - } -} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryErrorCode.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryErrorCode.java deleted file mode 100644 index 8ec04599d7891..0000000000000 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryErrorCode.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.opentelemetry; - -import com.facebook.presto.common.ErrorCode; -import com.facebook.presto.common.ErrorType; -import com.facebook.presto.spi.ErrorCodeSupplier; - -import static com.facebook.presto.common.ErrorType.EXTERNAL; - -public enum OpenTelemetryErrorCode - implements ErrorCodeSupplier -{ - OPEN_TELEMETRY_CONTEXT_PROPAGATOR_ERROR(0, EXTERNAL); - - private final ErrorCode errorCode; - - OpenTelemetryErrorCode(int code, ErrorType type) - { - errorCode = new ErrorCode(code + 0x0507_0000, name(), type); - } - - @Override - public ErrorCode toErrorCode() - { - return errorCode; - } -} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracer.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracer.java deleted file mode 100644 index 2538a8caa500b..0000000000000 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracer.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.opentelemetry; - -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.tracing.Tracer; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.baggage.Baggage; -import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; -import io.opentelemetry.context.propagation.TextMapGetter; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static com.facebook.presto.spi.StandardErrorCode.DISTRIBUTED_TRACING_ERROR; - -public class OpenTelemetryTracer - implements Tracer -{ - private static String currentContextPropagator = OpenTelemetryContextPropagator.B3_SINGLE_HEADER; - private static OpenTelemetry openTelemetry = OpenTelemetryBuilder.build(currentContextPropagator); - private final io.opentelemetry.api.trace.Tracer openTelemetryTracer; - private final String traceToken; - private final Span parentSpan; - private final Context parentContext; - - public final Map spanMap = new ConcurrentHashMap<>(); - public final Map recorderSpanMap = new LinkedHashMap<>(); - - public OpenTelemetryTracer(String traceToken, String contextPropagator, String propagatedContext, String baggage) - { - // Trivial getter method to return carrier - // Carrier will be equal to the data to be fetched - TextMapGetter trivialGetter = new TextMapGetter() - { - @Override - public String get(String carrier, String key) - { - return carrier; - } - - @Override - public Iterable keys(String carrier) - { - return Arrays.asList(get(carrier, null)); - } - }; - - // Rebuild OPEN_TELEMETRY instance if necessary (to use different context propagator) - // Will only occur once at max, if contextPropagator is different from B3_SINGLE_HEADER - if (contextPropagator != null && !contextPropagator.equals(currentContextPropagator)) { - this.openTelemetry = OpenTelemetryBuilder.build(contextPropagator); - this.currentContextPropagator = contextPropagator; - } - - this.openTelemetryTracer = openTelemetry.getTracer(tracerName); - this.traceToken = traceToken; - - if (propagatedContext != null) { - // Only process baggage headers if context propagation is successful - Context extractedContext = openTelemetry.getPropagators().getTextMapPropagator() - .extract(Context.current(), propagatedContext, trivialGetter); - Context contextWithBaggage = W3CBaggagePropagator.getInstance().extract( - extractedContext, baggage, trivialGetter); - try (Scope ignored = contextWithBaggage.makeCurrent()) { - this.parentSpan = createParentSpan(); - } - this.parentContext = contextWithBaggage; - } - else { - this.parentSpan = createParentSpan(); - this.parentContext = Context.current(); - } - addBaggageToSpanAttributes(this.parentSpan); - - synchronized (recorderSpanMap) { - recorderSpanMap.put("Trace start", this.parentSpan); - } - } - - /** - * Take parent context baggage and set as span attributes. - * Call during each span and nested span creation to properly propagate tags. - */ - private void addBaggageToSpanAttributes(Span span) - { - Baggage baggage = Baggage.fromContext(parentContext); - baggage.forEach((s, baggageEntry) -> span.setAttribute(s, baggageEntry.getValue())); - } - - private Span createParentSpan() - { - Span parentSpan = openTelemetryTracer.spanBuilder("Trace start").startSpan(); - parentSpan.setAttribute("trace_id", traceToken); - return parentSpan; - } - - private void endUnendedBlocks() - { - List blocks = new ArrayList<>(spanMap.keySet()); - for (String currBlock : blocks) { - endBlock(currBlock, ""); - } - } - - /** - * Add annotation as event to parent span - * @param annotation event to add - */ - @Override - public void addPoint(String annotation) - { - parentSpan.addEvent(annotation); - } - - /** - * Create new span with Open Telemetry tracer - * @param blockName name of span - * @param annotation event to add to span - */ - - @Override - public void startBlock(String blockName, String annotation) - { - if (spanMap.containsKey(blockName)) { - throw new PrestoException(DISTRIBUTED_TRACING_ERROR, "Duplicated block inserted: " + blockName); - } - Span span = openTelemetryTracer.spanBuilder(blockName) - .setParent(Context.current().with(parentSpan)) - .startSpan(); - span.addEvent(annotation); - addBaggageToSpanAttributes(span); - - spanMap.put(blockName, span); - synchronized (recorderSpanMap) { - recorderSpanMap.put(blockName, span); - } - } - - @Override - public void addPointToBlock(String blockName, String annotation) - { - if (!spanMap.containsKey(blockName)) { - throw new PrestoException(DISTRIBUTED_TRACING_ERROR, "Adding point to non-existing block: " + blockName); - } - spanMap.get(blockName).addEvent(annotation); - } - - /** - * End Open Telemetry span - * @param blockName name of span - * @param annotation event to add to span - */ - @Override - public void endBlock(String blockName, String annotation) - { - if (!spanMap.containsKey(blockName)) { - throw new PrestoException(DISTRIBUTED_TRACING_ERROR, "Trying to end a non-existing block: " + blockName); - } - spanMap.remove(blockName); - synchronized (recorderSpanMap) { - Span span = recorderSpanMap.get(blockName); - span.addEvent(annotation); - span.end(); - } - } - - @Override - public void endTrace(String annotation) - { - parentSpan.addEvent(annotation); - endUnendedBlocks(); - parentSpan.end(); - } - - @Override - public String getTracerId() - { - if (traceToken != null) { - return traceToken; - } - return tracerName; - } -} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracerHandle.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracerHandle.java deleted file mode 100644 index d68945b10761d..0000000000000 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracerHandle.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.opentelemetry; - -import com.facebook.presto.spi.tracing.TracerHandle; - -public class OpenTelemetryTracerHandle - implements TracerHandle -{ - private final String traceToken; - private final String contextPropagator; - private final String propagatedContext; - private final String baggage; - - public OpenTelemetryTracerHandle(String traceToken, String contextPropagator, String propagatedContext, String baggage) - { - this.traceToken = traceToken; - this.contextPropagator = contextPropagator; - this.propagatedContext = propagatedContext; - this.baggage = baggage; - } - - @Override - public String getTraceToken() - { - return traceToken; - } - - public String getContextPropagator() - { - return contextPropagator; - } - - public String getPropagatedContext() - { - return propagatedContext; - } - - public String getBaggage() - { - return baggage; - } -} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracerProvider.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracerProvider.java deleted file mode 100644 index 722b60937938d..0000000000000 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracerProvider.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.opentelemetry; - -import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.tracing.Tracer; -import com.facebook.presto.spi.tracing.TracerHandle; -import com.facebook.presto.spi.tracing.TracerProvider; -import com.google.inject.Inject; - -import java.util.Map; -import java.util.function.Function; - -import static com.facebook.presto.opentelemetry.OpenTelemetryErrorCode.OPEN_TELEMETRY_CONTEXT_PROPAGATOR_ERROR; -import static com.facebook.presto.opentelemetry.OpenTelemetryHeaders.PRESTO_B3_SINGLE_HEADER_PROPAGATION; -import static com.facebook.presto.opentelemetry.OpenTelemetryHeaders.PRESTO_BAGGAGE_HEADER; -import static com.facebook.presto.opentelemetry.OpenTelemetryHeaders.PRESTO_TRACE_TOKEN; -import static com.facebook.presto.opentelemetry.OpenTelemetryHeaders.PRESTO_W3C_PROPAGATION; - -public class OpenTelemetryTracerProvider - implements TracerProvider -{ - @Inject - public OpenTelemetryTracerProvider() {} - - @Override - public String getName() - { - return "Open telemetry tracer provider"; - } - - @Override - public String getTracerType() - { - return "otel"; - } - - @Override - public Function, TracerHandle> getHandleGenerator() - { - return headers -> { - String contextPropagator = determineContextPropagationMode(headers); - return new OpenTelemetryTracerHandle( - headers.get(PRESTO_TRACE_TOKEN), - contextPropagator, - getPropagatedContextFromHeader(contextPropagator, headers), - headers.get(PRESTO_BAGGAGE_HEADER)); - }; - } - - @Override - public Tracer getNewTracer(TracerHandle handle) - { - OpenTelemetryTracerHandle tracerHandle = (OpenTelemetryTracerHandle) handle; - return new OpenTelemetryTracer( - tracerHandle.getTraceToken(), - tracerHandle.getContextPropagator(), - tracerHandle.getPropagatedContext(), - tracerHandle.getBaggage()); - } - - /** - * Take header values and determine which context propagation mode is used - * Currently only supports b3 single header - * @param headers HTTP request headers - * @return context propagation mode - */ - private String determineContextPropagationMode(Map headers) - { - if (headers.containsKey(PRESTO_B3_SINGLE_HEADER_PROPAGATION)) { - return OpenTelemetryContextPropagator.B3_SINGLE_HEADER; - } - if (headers.containsKey(PRESTO_W3C_PROPAGATION)) { - throw new PrestoException( - OPEN_TELEMETRY_CONTEXT_PROPAGATOR_ERROR, - "Only b3 single header context propagation mode is currently supported."); - } - return null; - } - - /** - * Currently only supports b3 single header and w3c propagation - * @param contextPropagator context propagator to use - * @param headers http request headers - * @return header value extracted from http request based on context propagator - */ - private static String getPropagatedContextFromHeader(String contextPropagator, Map headers) - { - if (contextPropagator != null) { - if (contextPropagator.equals(OpenTelemetryContextPropagator.B3_SINGLE_HEADER)) { - return headers.get(PRESTO_B3_SINGLE_HEADER_PROPAGATION); - } - if (contextPropagator.equals(OpenTelemetryContextPropagator.W3C)) { - return headers.get(PRESTO_W3C_PROPAGATION); - } - } - return null; - } -} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracingImpl.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracingImpl.java new file mode 100644 index 0000000000000..45081d4dff8a2 --- /dev/null +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryTracingImpl.java @@ -0,0 +1,514 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.opentelemetry; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.ErrorCode; +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.common.telemetry.tracing.TracingEnum; +import com.facebook.presto.opentelemetry.tracing.ScopedSpan; +import com.facebook.presto.opentelemetry.tracing.TracingSpan; +import com.facebook.presto.spi.telemetry.TelemetryTracing; +import com.google.errorprone.annotations.MustBeClosed; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Strings.nullToEmpty; + +/** + * Open Telemetry implementation of tracing. + */ +public class OpenTelemetryTracingImpl + implements TelemetryTracing +{ + private static final Logger log = Logger.get(OpenTelemetryTracingImpl.class); + private static OpenTelemetry configuredOpenTelemetry; + private static Tracer tracer = OpenTelemetry.noop().getTracer("no-op"); //default tracer + + /** + * called from TracingManager for setting the open telemetry sdk and tracer. + */ + @Override + public void loadConfiguredOpenTelemetry() + { + log.debug("creating opentelemetry instance"); + configuredOpenTelemetry = createOpenTelemetry(); + + log.debug("creating telemetry tracer"); + createTracer(); + } + + /** + * creates and updates sdk tracer instance if tracing is enabled. Else uses a default no-op instance. + */ + public void createTracer() + { + if (TelemetryConfig.getTracingEnabled()) { + tracer = configuredOpenTelemetry.getTracer("Presto"); + } + } + + /** + * Create opentelemetry instance + * + * @return {@link OpenTelemetry} + */ + public OpenTelemetry createOpenTelemetry() + { + TelemetryConfig telemetryConfig = TelemetryConfig.getTelemetryConfig(); + OpenTelemetry openTelemetry = OpenTelemetry.noop(); //default instance for tracing disabled case + + if (TelemetryConfig.getTracingEnabled()) { + log.debug("telemetry tracing is enabled"); + Resource resource = Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), "Presto")); + + SpanExporter spanExporter = OtlpGrpcSpanExporter.builder() + .setEndpoint(telemetryConfig.getTracingBackendUrl()) + .setTimeout(10, TimeUnit.SECONDS) + .build(); + log.debug("telemetry span exporter configured"); + + SpanProcessor spanProcessor = BatchSpanProcessor.builder(spanExporter) + .setMaxExportBatchSize(telemetryConfig.getMaxExporterBatchSize()) + .setMaxQueueSize(telemetryConfig.getMaxQueueSize()) + .setScheduleDelay(telemetryConfig.getScheduleDelay(), TimeUnit.MILLISECONDS) + .setExporterTimeout(telemetryConfig.getExporterTimeout(), TimeUnit.MILLISECONDS) + .build(); + log.debug("telemetry span processor configured"); + + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .setSampler(Sampler.traceIdRatioBased(telemetryConfig.getSamplingRatio())) + .addSpanProcessor(spanProcessor) + .setResource(resource) + .build(); + log.debug("telemetry tracer provider set"); + + openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setPropagators(ContextPropagators.create( + TextMapPropagator.composite(W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance()))) + .build(); + log.debug("opentelemetry instance created"); + } + else { + log.debug("telemetry tracing is disabled"); + } + + return openTelemetry; + } + + /** + * Sets open telemetry. + * + * @param configuredOpenTelemetry the configured open telemetry + */ + public static void setOpenTelemetry(OpenTelemetry configuredOpenTelemetry) + { + OpenTelemetryTracingImpl.configuredOpenTelemetry = configuredOpenTelemetry; + } + + /** + * Sets tracer. + * + * @param tracer the tracer + */ + public static void setTracer(Tracer tracer) + { + OpenTelemetryTracingImpl.tracer = tracer; + } + + /** + * get current context wrapped . + * + * @param runnable runnable + * @return Runnable + */ + @Override + public Runnable getCurrentContextWrap(Runnable runnable) + { + return Context.current().wrap(runnable); + } + + /** + * get current context . + * + * @return Context + */ + private static Context getCurrentContext() + { + return Context.current(); + } + + /** + * get context from the span. + * + * @param tracingSpan runnable + * @return Context + */ + private static Context getCurrentContextWith(TracingSpan tracingSpan) + { + return Context.current().with(tracingSpan.getSpan()); + } + + /** + * get the context from the span or current context. + * + * @param span span + * @return Context + */ + private static Context getContext(TracingSpan span) + { + return span != null ? getCurrentContextWith(span) : getCurrentContext(); + } + + /** + * get the context from the traceParent string. + * + * @param traceParent traceParent + * @return Context + */ + private static Context getContext(String traceParent) + { + TextMapPropagator propagator = configuredOpenTelemetry.getPropagators().getTextMapPropagator(); + return propagator.extract(Context.current(), traceParent, new TextMapGetterImpl()); + } + + /** + * returns true if the span records tracing events. + * + * @return boolean + */ + @Override + public boolean isRecording() + { + return Span.fromContext(getCurrentContext()).isRecording(); + } + + /** + * returns headers map from the input span. + * + * @param span span + * @return Map + */ + @Override + public Map getHeadersMap(TracingSpan span) + { + TextMapPropagator propagator = configuredOpenTelemetry.getPropagators().getTextMapPropagator(); + Map headersMap = new HashMap<>(); + Context context = (span != null) ? Context.current().with(span.getSpan()) : Context.current(); + Context currentContext = (TelemetryConfig.getTracingEnabled()) ? context : null; + propagator.inject(currentContext, headersMap, Map::put); + return headersMap; + } + + /** + * ends span on error with recorded exceptions. + * + * @param span querySpan + * @param throwable throwable + * @return Map + */ + @Override + public void endSpanOnError(TracingSpan span, Throwable throwable) + { + if (TelemetryConfig.getTracingEnabled() && Objects.nonNull(span)) { + span.getSpan().setStatus(StatusCode.ERROR, throwable.getMessage()) + .recordException(throwable) + .end(); + } + } + + /** + * add event to the span. + * + * @param span span + * @param eventName eventName + */ + @Override + public void addEvent(TracingSpan span, String eventName) + { + if (TelemetryConfig.getTracingEnabled() && Objects.nonNull(span)) { + span.getSpan().addEvent(eventName); + } + } + + /** + * add event to the span. + * + * @param span span + * @param eventName eventName + * @param eventState eventState + */ + @Override + public void addEvent(TracingSpan span, String eventName, String eventState) + { + if (TelemetryConfig.getTracingEnabled() && Objects.nonNull(span)) { + span.getSpan().addEvent(eventName, Attributes.of(AttributeKey.stringKey("EVENT_STATE"), eventState)); + } + } + + /** + * set attributes to the span. + * + * @param span span + * @param attributes attributes + */ + @Override + public void setAttributes(TracingSpan span, Map attributes) + { + if (TelemetryConfig.getTracingEnabled() && Objects.nonNull(span)) { + attributes.forEach(span::setAttribute); + } + } + + /** + * record exception to the span. + * + * @param span span + * @param message message + * @param runtimeException runtimeException + * @param errorCode errorCode + */ + @Override + public void recordException(TracingSpan span, String message, RuntimeException runtimeException, ErrorCode errorCode) + { + if (TelemetryConfig.getTracingEnabled() && Objects.nonNull(span)) { + span.getSpan().setStatus(StatusCode.ERROR, nullToEmpty(message)) + .recordException(runtimeException) + .setAttribute("ERROR_CODE", errorCode.getCode()) + .setAttribute("ERROR_NAME", errorCode.getName()) + .setAttribute("ERROR_TYPE", errorCode.getType().toString()); + } + } + + /** + * mark success to span. + * + * @param span span + */ + @Override + public void setSuccess(TracingSpan span) + { + if (TelemetryConfig.getTracingEnabled() && Objects.nonNull(span)) { + span.getSpan().setStatus(StatusCode.OK); + } + } + + //Tracing spans + /** + * To get an invalid span. + * @return TracingSpan + */ + @Override + public TracingSpan getInvalidSpan() + { + return new TracingSpan(Span.getInvalid()); + } + + /** + * To get root span. + * @return TracingSpan + */ + @Override + public TracingSpan getRootSpan() + { + return !TelemetryConfig.getTracingEnabled() ? new TracingSpan(Span.getInvalid()) : new TracingSpan(tracer.spanBuilder(TracingEnum.ROOT.getName()).setSpanKind(SpanKind.SERVER) + .startSpan()); + } + + /** + * To get span with name. + * @param spanName name of span to be created + * @return TracingSpan + */ + @Override + public TracingSpan getSpan(String spanName) + { + return !TelemetryConfig.getTracingEnabled() ? new TracingSpan(Span.getInvalid()) : new TracingSpan(tracer.spanBuilder(spanName) + .startSpan()); + } + + /** + * To get a new span with name from the trace parent string. + * @param traceParent trace parent string. + * @param spanName name of the span to be created. + * @return TracingSpan + */ + @Override + public TracingSpan getSpan(String traceParent, String spanName) + { + return !TelemetryConfig.getTracingEnabled() || traceParent == null ? new TracingSpan(Span.getInvalid()) : new TracingSpan(tracer.spanBuilder(spanName) + .setParent(getContext(traceParent)) + .startSpan()); + } + + /** + * To get a new span with name from the parent span. + * @param parentSpan parent span. + * @param spanName name of the span to be created. + * @param attributes input attributes to set in span. + * @return TracingSpan + */ + @Override + public TracingSpan getSpan(TracingSpan parentSpan, String spanName, Map attributes) + { + return !TelemetryConfig.getTracingEnabled() ? new TracingSpan(Span.getInvalid()) : new TracingSpan(setAttributes(tracer.spanBuilder(spanName), attributes) + .setParent(getContext(parentSpan)) + .startSpan()); + } + + /** + * To set the input attributes in span builder. + * @param spanBuilder span builder. + * @param attributes input attributes to set in span builder. + * @return SpanBuilder + */ + private static SpanBuilder setAttributes(SpanBuilder spanBuilder, Map attributes) + { + attributes.forEach(spanBuilder::setAttribute); + return spanBuilder; + } + + /** + * To get the string value of the input span. + * @param span input span. + * @return Optional + */ + @Override + public Optional spanString(TracingSpan span) + { + return Optional.ofNullable(span) + .filter(s -> span.getSpan().getSpanContext().isValid()) + .map(s -> toStringHelper("Span") + .add("spanId", span.getSpan().getSpanContext().getSpanId()) + .add("traceId", span.getSpan().getSpanContext().getTraceId()) + .toString()); + } + + //Scoped Spans + /** + * To get ScopedSpan with name. + * @param spanName name of span to be created + * @param skipSpan optional parameter to implement span sampling by skipping the current span export + * @return ScopedSpan + */ + @MustBeClosed + @Override + public ScopedSpan scopedSpan(String spanName, Boolean... skipSpan) + { + if (!TelemetryConfig.getTracingEnabled() || (skipSpan.length > 0 && TelemetryConfig.getSpanSampling())) { + return null; + } + return scopedSpan(new TracingSpan(tracer.spanBuilder(spanName).startSpan())); + } + + /** + * To get ScopedSpan from the parent span instance. + * @param parentSpan parent span + * @param skipSpan optional parameter to implement span sampling by skipping the current span export + * @return ScopedSpan + */ + @MustBeClosed + @Override + public ScopedSpan scopedSpan(TracingSpan parentSpan, Boolean... skipSpan) + { + if ((!TelemetryConfig.getTracingEnabled() || Objects.isNull(parentSpan)) || (skipSpan.length > 0 && TelemetryConfig.getSpanSampling())) { + return null; + } + return new ScopedSpan(parentSpan.getSpan()); + } + + /** + * To get ScopedSpan from the parent span instance with name and also setting the input attributes. + * @param parentSpan parent span + * @param skipSpan optional parameter to implement span sampling by skipping the current span export + * @return ScopedSpan + */ + @MustBeClosed + @Override + public ScopedSpan scopedSpan(TracingSpan parentSpan, String spanName, Map attributes, Boolean... skipSpan) + { + if (!TelemetryConfig.getTracingEnabled() || (skipSpan.length > 0 && TelemetryConfig.getSpanSampling())) { + return null; + } + SpanBuilder spanBuilder = tracer.spanBuilder(spanName); + Span span = setAttributes(spanBuilder, attributes) + .setParent(getContext(parentSpan)) + .startSpan(); + return new ScopedSpan(span); + } + + /** + * To get ScopedSpan from the parent span instance with name and also setting the input attributes. + * @param parentSpan parent span + * @param skipSpan optional parameter to implement span sampling by skipping the current span export + * @return ScopedSpan + */ + @MustBeClosed + @Override + public ScopedSpan scopedSpan(TracingSpan parentSpan, String spanName, Boolean... skipSpan) + { + if (!TelemetryConfig.getTracingEnabled() || (skipSpan.length > 0 && TelemetryConfig.getSpanSampling())) { + return null; + } + Span span = tracer.spanBuilder(spanName) + .setParent(getContext(parentSpan)) + .startSpan(); + return new ScopedSpan(span); + } + + /** + * To get ScopedSpan with name and also setting the input attributes. + * @param spanName span name + * @param skipSpan optional parameter to implement span sampling by skipping the current span export + * @return ScopedSpan + */ + @MustBeClosed + @Override + public ScopedSpan scopedSpan(String spanName, Map attributes, Boolean... skipSpan) + { + if (!TelemetryConfig.getTracingEnabled() || (skipSpan.length > 0 && TelemetryConfig.getSpanSampling())) { + return null; + } + SpanBuilder spanBuilder = tracer.spanBuilder(spanName); + Span span = setAttributes(spanBuilder, attributes).startSpan(); + return new ScopedSpan(span); + } +} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryHeaders.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/TextMapGetterImpl.java similarity index 54% rename from presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryHeaders.java rename to presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/TextMapGetterImpl.java index 4e80c84c7c041..4fdff0962471b 100644 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryHeaders.java +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/TextMapGetterImpl.java @@ -13,12 +13,28 @@ */ package com.facebook.presto.opentelemetry; -public class OpenTelemetryHeaders +import io.opentelemetry.context.propagation.TextMapGetter; + +import java.util.Collections; + +/** + * The type Text map getter. + */ +public class TextMapGetterImpl + implements TextMapGetter { - public static final String PRESTO_W3C_PROPAGATION = "traceparent"; - public static final String PRESTO_B3_SINGLE_HEADER_PROPAGATION = "b3"; - public static final String PRESTO_TRACE_TOKEN = "X-Presto-Trace-Token"; - public static final String PRESTO_BAGGAGE_HEADER = "baggage"; + @Override + public Iterable keys(Object carrier) + { + return Collections.singletonList("traceparent"); + } - private OpenTelemetryHeaders() {} + @Override + public String get(Object carrier, String key) + { + if ("traceparent".equals(key)) { + return (String) carrier; + } + return null; + } } diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/OpenTelemetryFactoryImpl.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/OpenTelemetryFactoryImpl.java new file mode 100644 index 0000000000000..fbb3b0f103378 --- /dev/null +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/OpenTelemetryFactoryImpl.java @@ -0,0 +1,45 @@ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.opentelemetry.tracing; + +import com.facebook.presto.opentelemetry.OpenTelemetryTracingImpl; +import com.facebook.presto.spi.telemetry.TelemetryFactory; +import com.facebook.presto.spi.telemetry.TelemetryTracing; + +public class OpenTelemetryFactoryImpl + implements TelemetryFactory> +{ + /** + * uniquely identify all OpenTelemetryFactory implementations. This property is checked against the one passed in + * telemetry.properties file during registration + * @return String + */ + @Override + public String getName() + { + return "otel"; + } + + /** + * Create OpenTelemetryImpl instance + * + * @return {@link OpenTelemetryTracingImpl} + */ + @Override + public OpenTelemetryTracingImpl create() + { + return new OpenTelemetryTracingImpl(); + } +} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/ScopedSpan.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/ScopedSpan.java new file mode 100644 index 0000000000000..2f95dcde144b9 --- /dev/null +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/ScopedSpan.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.opentelemetry.tracing; + +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.spi.telemetry.BaseSpan; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; + +import java.util.Objects; + +/** + * The type Scoped span. + */ +public final class ScopedSpan + implements BaseSpan +{ + private final Span span; + private final Scope scope; + + /** + * Instantiates a new Scoped span. + * + * @param span the span + */ + @SuppressWarnings("MustBeClosedChecker") + public ScopedSpan(Span span) + { + this.span = span; + this.scope = span.makeCurrent(); + } + + @Override + public void close() + { + if (!TelemetryConfig.getTracingEnabled()) { + return; + } + try { + scope.close(); + } + finally { + if (!Objects.isNull(span)) { + span.end(); + } + } + } + + @Override + public void end() + { + span.end(); + } +} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/TracingSpan.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/TracingSpan.java new file mode 100644 index 0000000000000..3c27a93263ae0 --- /dev/null +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/tracing/TracingSpan.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.opentelemetry.tracing; + +import com.facebook.presto.spi.telemetry.BaseSpan; +import io.opentelemetry.api.trace.Span; + +/** + * The type Tracing span. + */ +public class TracingSpan + implements BaseSpan +{ + private final Span span; + + /** + * Instantiates a new Tracing span. + * + * @param span the span + */ + public TracingSpan(Span span) + { + this.span = span; + } + + /** + * Gets span. + * + * @return the span + */ + public Span getSpan() + { + return span; + } + + /** + * Sets attribute. + * + * @param key the key + * @param value the value + */ + public void setAttribute(String key, String value) + { + span.setAttribute(key, value); + } + + @Override + public void end() + { + span.end(); + } +} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryPlugin.java b/presto-open-telemetry/src/main/java/com/facebook/presto/plugin/opentelemetry/OpenTelemetryPlugin.java similarity index 61% rename from presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryPlugin.java rename to presto-open-telemetry/src/main/java/com/facebook/presto/plugin/opentelemetry/OpenTelemetryPlugin.java index 807f1ba810f0b..123ab6634225b 100644 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryPlugin.java +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/plugin/opentelemetry/OpenTelemetryPlugin.java @@ -11,18 +11,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.opentelemetry; +package com.facebook.presto.plugin.opentelemetry; +import com.facebook.presto.opentelemetry.tracing.OpenTelemetryFactoryImpl; import com.facebook.presto.spi.Plugin; -import com.facebook.presto.spi.tracing.TracerProvider; +import com.facebook.presto.spi.telemetry.TelemetryFactory; +import com.facebook.presto.testing.TestingOpenTelemetryFactoryImpl; import com.google.common.collect.ImmutableList; +/** + * The type Open telemetry plugin. + */ public class OpenTelemetryPlugin implements Plugin { @Override - public Iterable getTracerProviders() + public Iterable getTelemetryFactories() { - return ImmutableList.of(new OpenTelemetryTracerProvider()); + return ImmutableList.of(new OpenTelemetryFactoryImpl(), new TestingOpenTelemetryFactoryImpl()); } } diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/testing/TestingOpenTelemetryFactoryImpl.java b/presto-open-telemetry/src/main/java/com/facebook/presto/testing/TestingOpenTelemetryFactoryImpl.java new file mode 100644 index 0000000000000..bac73dd25a94c --- /dev/null +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/testing/TestingOpenTelemetryFactoryImpl.java @@ -0,0 +1,45 @@ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.testing; + +import com.facebook.presto.opentelemetry.OpenTelemetryTracingImpl; +import com.facebook.presto.spi.telemetry.TelemetryFactory; +import com.facebook.presto.spi.testing.TestingTelemetryTracing; + +public class TestingOpenTelemetryFactoryImpl + implements TelemetryFactory +{ + /** + * uniquely identify all OpenTelemetryFactory implementations. This property is checked against the one passed in + * telemetry.properties file during registration + * @return String + */ + @Override + public String getName() + { + return "oteltest"; + } + + /** + * Create OpenTelemetryImpl instance + * + * @return {@link OpenTelemetryTracingImpl} + */ + @Override + public TestingOpenTelemetryTracing create() + { + return new TestingOpenTelemetryTracing(); + } +} diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/testing/TestingOpenTelemetryTracing.java b/presto-open-telemetry/src/main/java/com/facebook/presto/testing/TestingOpenTelemetryTracing.java new file mode 100644 index 0000000000000..ea3b516822af3 --- /dev/null +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/testing/TestingOpenTelemetryTracing.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.testing; + +import com.facebook.presto.common.TelemetryConfig; +import com.facebook.presto.opentelemetry.OpenTelemetryTracingImpl; +import com.facebook.presto.spi.testing.TestingTelemetryTracing; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; + +import java.util.List; + +public class TestingOpenTelemetryTracing + extends OpenTelemetryTracingImpl implements TestingTelemetryTracing +{ + private static OpenTelemetry openTelemetry = OpenTelemetry.noop(); + + private static Tracer tracer = openTelemetry.getTracer("no-op"); + + private static InMemorySpanExporter inMemorySpanExporter; + + @Override + public void loadConfiguredOpenTelemetry() + { + inMemorySpanExporter = InMemorySpanExporter.create(); + + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(BatchSpanProcessor.builder(inMemorySpanExporter) + .build()) + .setResource(Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), "Presto"))) + .build(); + + openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setPropagators(ContextPropagators.create( + TextMapPropagator.composite(W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance()))) + .build(); + + if (TelemetryConfig.getTracingEnabled()) { + tracer = openTelemetry.getTracer("sdk in mem tracer"); + OpenTelemetryTracingImpl.setOpenTelemetry(openTelemetry); + OpenTelemetryTracingImpl.setTracer(tracer); + } + } + + @Override + public List getFinishedSpanItems() + { + return inMemorySpanExporter.getFinishedSpanItems(); + } + + @Override + public boolean isSpansEmpty() + { + return getFinishedSpanItems().isEmpty(); + } + + @Override + public boolean spansAnyMatch(String spanName) + { + return getFinishedSpanItems().stream().anyMatch(sn -> spanName.equals(sn.getName())); + } + + @Override + public void clearSpanList() + { + inMemorySpanExporter.reset(); + } +} diff --git a/presto-open-telemetry/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin b/presto-open-telemetry/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin deleted file mode 100644 index 821a4cf114184..0000000000000 --- a/presto-open-telemetry/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin +++ /dev/null @@ -1 +0,0 @@ -com.facebook.presto.opentelemetry.OpenTelemetryPlugin \ No newline at end of file diff --git a/presto-open-telemetry/src/modernizer/violations.xml b/presto-open-telemetry/src/modernizer/violations.xml new file mode 100644 index 0000000000000..bca7ebcf3900a --- /dev/null +++ b/presto-open-telemetry/src/modernizer/violations.xml @@ -0,0 +1,45 @@ + + + + java/lang/Class.newInstance:()Ljava/lang/Object; + 1.1 + Prefer Class.getConstructor().newInstance() + + + + java/lang/String.toLowerCase:()Ljava/lang/String; + 1.1 + Prefer String.toLowerCase(java.util.Locale) + + + + com/google/common/primitives/Ints.checkedCast:(J)I + 1.8 + Prefer Math.toIntExact(long) + + + + org/testng/Assert.assertEquals:(Ljava/lang/Iterable;Ljava/lang/Iterable;)V + 1.8 + Use com.facebook.presto.testing.assertions.Assert.assertEquals due to TestNG #543 + + + + org/testng/Assert.assertEquals:(Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/String;)V + 1.8 + Use com.facebook.presto.testing.assertions.Assert.assertEquals due to TestNG #543 + + + + java/util/TimeZone.getTimeZone:(Ljava/lang/String;)Ljava/util/TimeZone; + 1.8 + Avoid TimeZone.getTimeZone as it returns GMT for a zone not supported by the JVM. Use TimeZone.getTimeZone(ZoneId.of(..)) instead, or TimeZone.getTimeZone(..., false). + + + + org/joda/time/DateTimeZone.toTimeZone:()Ljava/util/TimeZone; + 1.8 + Avoid DateTimeZone.toTimeZone as it returns GMT for a zone not supported by the JVM. Use TimeZone.getTimeZone(ZoneId.of(dtz.getId())) instead. + + + diff --git a/presto-open-telemetry/src/test/java/com/facebook/presto/opentelemetry/TestOpenTelemetryTracingImpl.java b/presto-open-telemetry/src/test/java/com/facebook/presto/opentelemetry/TestOpenTelemetryTracingImpl.java new file mode 100644 index 0000000000000..689826b39c1c1 --- /dev/null +++ b/presto-open-telemetry/src/test/java/com/facebook/presto/opentelemetry/TestOpenTelemetryTracingImpl.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.opentelemetry; + +import com.facebook.presto.common.TelemetryConfig; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class TestOpenTelemetryTracingImpl +{ + private OpenTelemetryTracingImpl openTelemetryTracingImpl; + Map properties = new HashMap<>(); + + @BeforeMethod + public void setUp() throws Exception + { + openTelemetryTracingImpl = new OpenTelemetryTracingImpl(); + resetTelemetryConfigSingleton(); + } + + @Test + public void testCreateWithTracingEnabled() + { + properties.put("tracing-enabled", "true"); + properties.put("tracing-backend-url", "http://localhost:4317"); + properties.put("max-exporter-batch-size", "256"); + properties.put("max-queue-size", "1024"); + properties.put("exporter-timeout", "5000"); + properties.put("schedule-delay", "1000"); + properties.put("trace-sampling-ratio", "1.0"); + properties.put("span-sampling", "true"); + + TelemetryConfig.getTelemetryConfig().setTelemetryProperties(properties); + OpenTelemetry openTelemetry = openTelemetryTracingImpl.createOpenTelemetry(); + + assertNotNull(openTelemetry); + assertTrue(openTelemetry instanceof OpenTelemetrySdk, "sdk instance"); + } + + @Test + public void testCreateWithTracingDisabled() + { + TelemetryConfig.getTelemetryConfig().setTracingEnabled(false); + OpenTelemetry openTelemetry = openTelemetryTracingImpl.createOpenTelemetry(); + + assertNotNull(openTelemetry); + assertEquals(openTelemetry, OpenTelemetry.noop(), "no-op instance"); + } + + private void resetTelemetryConfigSingleton() throws Exception + { + Field instanceField = TelemetryConfig.class.getDeclaredField("telemetryConfig"); + instanceField.setAccessible(true); + instanceField.set(null, null); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java index a245586ddf043..b0bf32c72b4a6 100644 --- a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSplitManager.java @@ -227,7 +227,6 @@ public static ConnectorSession createSessionWithNumSplits(int numSegmentsPerSpli "user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), - Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), @@ -250,7 +249,6 @@ public static ConnectorSession createSessionWithLimitLarge(int limitLarge, Pinot "user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), - Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), @@ -271,7 +269,6 @@ public static ConnectorSession createSessionWithTopNLarge(int topNLarge, PinotCo "user", new ConnectorIdentity("user", Optional.empty(), Optional.empty()), Optional.of("test"), - Optional.empty(), UTC_KEY, ENGLISH, System.currentTimeMillis(), diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index e9d03e52a461f..15c6907444ae6 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -199,8 +199,7 @@ import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManagerConfig; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; -import com.facebook.presto.tracing.TracerProviderManager; -import com.facebook.presto.tracing.TracingConfig; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.transaction.InMemoryTransactionManager; import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.transaction.TransactionManagerConfig; @@ -276,7 +275,6 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(SqlEnvironmentConfig.class); configBinder(binder).bindConfig(StaticFunctionNamespaceStoreConfig.class); configBinder(binder).bindConfig(PrestoSparkConfig.class); - configBinder(binder).bindConfig(TracingConfig.class); configBinder(binder).bindConfig(NativeExecutionVeloxConfig.class); configBinder(binder).bindConfig(NativeExecutionSystemConfig.class); configBinder(binder).bindConfig(NativeExecutionNodeConfig.class); @@ -353,9 +351,6 @@ protected void setup(Binder binder) // expression manager binder.bind(ExpressionOptimizerManager.class).in(Scopes.SINGLETON); - // tracer provider managers - binder.bind(TracerProviderManager.class).in(Scopes.SINGLETON); - // block encodings binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON); binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON); @@ -563,6 +558,7 @@ protected void setup(Binder binder) binder.bind(ClientRequestFilterManager.class).in(Scopes.SINGLETON); binder.bind(PlanCheckerProviderManager.class).in(Scopes.SINGLETON); binder.bind(NodeManager.class).to(PluginNodeManager.class).in(Scopes.SINGLETON); + binder.bind(TracingManager.class).in(Scopes.SINGLETON); } @Provides diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java index 850068ac5d722..9ed653175c6e8 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java @@ -102,6 +102,7 @@ import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.storage.TempStorageManager; +import com.facebook.presto.telemetry.TracingManager; import com.facebook.presto.transaction.TransactionManager; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ArrayListMultimap; @@ -614,7 +615,7 @@ public IPrestoSparkQueryExecution create( credentialsProviders, authenticatorProviders); - SessionBuilder sessionBuilder = sessionSupplier.createSessionBuilder(queryId, sessionContext, warningCollectorFactory); + SessionBuilder sessionBuilder = sessionSupplier.createSessionBuilder(queryId, TracingManager.getInvalidSpan(), TracingManager.getInvalidSpan(), sessionContext, warningCollectorFactory); sessionPropertyDefaults.applyDefaultProperties(sessionBuilder, Optional.empty(), Optional.empty()); if (!executionStrategies.isEmpty()) { diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java index 49ac83c213b9d..f1d224591616a 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java @@ -24,7 +24,6 @@ import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.security.TokenAuthenticator; import com.facebook.presto.spi.session.ResourceEstimates; -import com.facebook.presto.spi.tracing.Tracer; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -52,7 +51,6 @@ public class PrestoSparkSessionContext private final Map systemProperties; private final Map> catalogSessionProperties; - private final Optional traceToken; private final RuntimeStats runtimeStats = new RuntimeStats(); public static PrestoSparkSessionContext createFromSessionInfo( @@ -85,8 +83,7 @@ public static PrestoSparkSessionContext createFromSessionInfo( prestoSparkSession.getTimeZoneId().orElse(null), prestoSparkSession.getLanguage().orElse(null), prestoSparkSession.getSystemProperties(), - prestoSparkSession.getCatalogSessionProperties(), - prestoSparkSession.getTraceToken()); + prestoSparkSession.getCatalogSessionProperties()); } public PrestoSparkSessionContext( @@ -100,8 +97,7 @@ public PrestoSparkSessionContext( String timeZoneId, String language, Map systemProperties, - Map> catalogSessionProperties, - Optional traceToken) + Map> catalogSessionProperties) { this.identity = requireNonNull(identity, "identity is null"); this.catalog = catalog; @@ -114,7 +110,6 @@ public PrestoSparkSessionContext( this.language = language; this.systemProperties = ImmutableMap.copyOf(requireNonNull(systemProperties, "systemProperties is null")); this.catalogSessionProperties = ImmutableMap.copyOf(requireNonNull(catalogSessionProperties, "catalogSessionProperties is null")); - this.traceToken = requireNonNull(traceToken, "traceToken is null"); } @Override @@ -191,12 +186,6 @@ public String getLanguage() return language; } - @Override - public Optional getTracer() - { - return Optional.empty(); - } - @Override public Map getSystemProperties() { @@ -223,12 +212,6 @@ public Optional getTransactionId() return Optional.empty(); } - @Override - public Optional getTraceToken() - { - return traceToken; - } - @Override public boolean supportClientTransaction() { diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecution.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecution.java index 26ecc79195412..eb84e44e53411 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecution.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkTaskExecution.java @@ -270,7 +270,7 @@ public void onSuccess(Object result) checkTaskCompletion(); - splitMonitor.splitCompletedEvent(taskId, getDriverStats()); + splitMonitor.splitCompletedEvent(taskId, getDriverStats(), null); } } @@ -284,7 +284,7 @@ public void onFailure(Throwable cause) remainingDrivers.decrementAndGet(); // fire failed event with cause - splitMonitor.splitFailedEvent(taskId, getDriverStats(), cause); + splitMonitor.splitFailedEvent(taskId, getDriverStats(), cause, null); } } diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/node/PrestoSparkTaskManager.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/node/PrestoSparkTaskManager.java index cd4c3a457ccbe..e2dd7dc621ff9 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/node/PrestoSparkTaskManager.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/node/PrestoSparkTaskManager.java @@ -27,6 +27,7 @@ import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.memory.MemoryPoolAssignmentsRequest; import com.facebook.presto.metadata.MetadataUpdates; +import com.facebook.presto.spi.telemetry.BaseSpan; import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.util.concurrent.ListenableFuture; import io.airlift.units.DataSize; @@ -80,7 +81,7 @@ public void updateMemoryPoolAssignments(MemoryPoolAssignmentsRequest assignments } @Override - public TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, Optional tableWriteInfo) + public TaskInfo updateTask(Session session, TaskId taskId, Optional fragment, List sources, OutputBuffers outputBuffers, Optional tableWriteInfo, BaseSpan span) { throw new UnsupportedOperationException(); } diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java index c5adcf7f0efe2..179ab08cb1798 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java @@ -642,8 +642,7 @@ private static PrestoSparkSession createSessionInfo(Session session) Optional.of(session.getTimeZoneKey().getId()), Optional.empty(), session.getSystemProperties(), - catalogSessionProperties.build(), - session.getTraceToken()); + catalogSessionProperties.build()); } @Override diff --git a/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java b/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java index 5fd5e354e0595..b5faa44c26935 100644 --- a/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java +++ b/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java @@ -43,7 +43,6 @@ public class PrestoSparkSession private final Optional language; private final Map systemProperties; private final Map> catalogSessionProperties; - private final Optional traceToken; public PrestoSparkSession( String user, @@ -58,8 +57,7 @@ public PrestoSparkSession( Optional timeZoneId, Optional language, Map systemProperties, - Map> catalogSessionProperties, - Optional traceToken) + Map> catalogSessionProperties) { this.user = requireNonNull(user, "user is null"); @@ -76,7 +74,6 @@ public PrestoSparkSession( this.systemProperties = unmodifiableMap(new HashMap<>(requireNonNull(systemProperties, "systemProperties is null"))); this.catalogSessionProperties = unmodifiableMap(requireNonNull(catalogSessionProperties, "catalogSessionProperties is null").entrySet().stream() .collect(toMap(Map.Entry::getKey, entry -> unmodifiableMap(new HashMap<>(entry.getValue()))))); - this.traceToken = requireNonNull(traceToken, "traceToken is null"); } public String getUser() @@ -143,9 +140,4 @@ public Map> getCatalogSessionProperties() { return catalogSessionProperties; } - - public Optional getTraceToken() - { - return traceToken; - } } diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java index 2748dcf919630..c9ad019425bdb 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java @@ -93,7 +93,6 @@ public void run() Optional.empty(), Optional.empty(), Optional.empty(), - Optional.empty(), Optional.empty()); } } diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java index acbf1568a7fcd..0647a3895ec53 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java @@ -94,7 +94,6 @@ public void run( Optional sqlLocation, Optional sqlFileHexHash, Optional sqlFileSizeInBytes, - Optional traceToken, Optional sparkQueueName, Optional queryStatusInfoOutputLocation, Optional queryDataOutputLocation) @@ -116,7 +115,6 @@ public void run( sqlLocation, sqlFileHexHash, sqlFileSizeInBytes, - traceToken, sparkQueueName, queryStatusInfoOutputLocation, queryDataOutputLocation, @@ -161,8 +159,7 @@ private void execute(IPrestoSparkQueryExecutionFactory queryExecutionFactory, Pr Optional.empty(), Optional.empty(), prestoSparkRunnerContext.getSessionProperties(), - prestoSparkRunnerContext.getCatalogSessionProperties(), - prestoSparkRunnerContext.getTraceToken()); + prestoSparkRunnerContext.getCatalogSessionProperties()); IPrestoSparkQueryExecution queryExecution = queryExecutionFactory.create( distribution.getSparkContext(), diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java index 1c0d8541e8bb8..1a36e3544afb9 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java @@ -40,7 +40,6 @@ public class PrestoSparkRunnerContext private final Optional sqlLocation; private final Optional sqlFileHexHash; private final Optional sqlFileSizeInBytes; - private final Optional traceToken; private final Optional sparkQueueName; private final Optional queryStatusInfoOutputLocation; private final Optional queryDataOutputLocation; @@ -62,7 +61,6 @@ public PrestoSparkRunnerContext( Optional sqlLocation, Optional sqlFileHexHash, Optional sqlFileSizeInBytes, - Optional traceToken, Optional sparkQueueName, Optional queryStatusInfoOutputLocation, Optional queryDataOutputLocation, @@ -83,7 +81,6 @@ public PrestoSparkRunnerContext( this.sqlLocation = sqlLocation; this.sqlFileHexHash = sqlFileHexHash; this.sqlFileSizeInBytes = sqlFileSizeInBytes; - this.traceToken = traceToken; this.sparkQueueName = sparkQueueName; this.queryStatusInfoOutputLocation = queryStatusInfoOutputLocation; this.queryDataOutputLocation = queryDataOutputLocation; @@ -165,11 +162,6 @@ public Optional getSqlFileSizeInBytes() return sqlFileSizeInBytes; } - public Optional getTraceToken() - { - return traceToken; - } - public Optional getSparkQueueName() { return sparkQueueName; @@ -207,7 +199,6 @@ public static class Builder private Optional sqlLocation; private Optional sqlFileHexHash; private Optional sqlFileSizeInBytes; - private Optional traceToken; private Optional sparkQueueName; private Optional queryStatusInfoOutputLocation; private Optional queryDataOutputLocation; @@ -230,7 +221,6 @@ public Builder(PrestoSparkRunnerContext prestoSparkRunnerContext) this.sqlLocation = prestoSparkRunnerContext.getSqlLocation(); this.sqlFileHexHash = prestoSparkRunnerContext.getSqlFileHexHash(); this.sqlFileSizeInBytes = prestoSparkRunnerContext.getSqlFileSizeInBytes(); - this.traceToken = prestoSparkRunnerContext.getTraceToken(); this.sparkQueueName = prestoSparkRunnerContext.getSparkQueueName(); this.queryStatusInfoOutputLocation = prestoSparkRunnerContext.getQueryStatusInfoOutputLocation(); this.queryDataOutputLocation = prestoSparkRunnerContext.getQueryDataOutputLocation(); @@ -261,7 +251,6 @@ public PrestoSparkRunnerContext build() sqlLocation, sqlFileHexHash, sqlFileSizeInBytes, - traceToken, sparkQueueName, queryStatusInfoOutputLocation, queryDataOutputLocation, diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java index a727e1f2f85f9..8b75f6ea67392 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/ConnectorSession.java @@ -42,8 +42,6 @@ default String getUser() Locale getLocale(); - Optional getTraceToken(); - Optional getClientInfo(); Set getClientTags(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java index 2da8ad1970eec..706f7933d4b55 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java @@ -30,7 +30,7 @@ import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; import com.facebook.presto.spi.storage.TempStorageFactory; -import com.facebook.presto.spi.tracing.TracerProvider; +import com.facebook.presto.spi.telemetry.TelemetryFactory; import com.facebook.presto.spi.ttl.ClusterTtlProviderFactory; import com.facebook.presto.spi.ttl.NodeTtlFetcherFactory; @@ -126,14 +126,6 @@ default Iterable getHistoryBasedPlanStatisti return emptyList(); } - /** - * Return list of tracer providers specified by tracer plugin - */ - default Iterable getTracerProviders() - { - return emptyList(); - } - default Iterable getAnalyzerProviders() { return emptyList(); @@ -153,4 +145,9 @@ default Iterable getClientRequestFilterFactories() { return emptyList(); } + + default Iterable getTelemetryFactories() + { + return emptyList(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java index ca96cdba0bb44..8b1cef902b6ab 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/eventlistener/QueryMetadata.java @@ -25,7 +25,6 @@ public class QueryMetadata { private final String queryId; private final Optional transactionId; - private final Optional tracingId; private final String query; private final String queryHash; @@ -55,8 +54,7 @@ public QueryMetadata( Optional jsonPlan, Optional graphvizPlan, Optional payload, - List runtimeOptimizedStages, - Optional tracingId) + List runtimeOptimizedStages) { this.queryId = requireNonNull(queryId, "queryId is null"); this.transactionId = requireNonNull(transactionId, "transactionId is null"); @@ -70,7 +68,6 @@ public QueryMetadata( this.graphvizPlan = requireNonNull(graphvizPlan, "graphvizPlan is null"); this.payload = requireNonNull(payload, "payload is null"); this.runtimeOptimizedStages = requireNonNull(runtimeOptimizedStages, "runtimeOptimizedStages is null"); - this.tracingId = requireNonNull(tracingId, "tracingId is null"); } @JsonProperty @@ -144,10 +141,4 @@ public List getRuntimeOptimizedStages() { return runtimeOptimizedStages; } - - @JsonProperty - public Optional getTracingId() - { - return tracingId; - } } diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryContextPropagator.java b/presto-spi/src/main/java/com/facebook/presto/spi/telemetry/BaseSpan.java similarity index 66% rename from presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryContextPropagator.java rename to presto-spi/src/main/java/com/facebook/presto/spi/telemetry/BaseSpan.java index 89e10342377fc..f7938a74ccaf4 100644 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryContextPropagator.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/telemetry/BaseSpan.java @@ -11,12 +11,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.opentelemetry; +package com.facebook.presto.spi.telemetry; -public final class OpenTelemetryContextPropagator +/** + * The interface Base span. + */ +public interface BaseSpan + extends AutoCloseable { - private OpenTelemetryContextPropagator() {}; + @Override + default void close() + { + return; + } - public static final String W3C = "w3c"; - public static final String B3_SINGLE_HEADER = "b3_single_header"; + /** + * End. + */ + default void end() + { + return; + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/TracerHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/telemetry/TelemetryFactory.java similarity index 63% rename from presto-spi/src/main/java/com/facebook/presto/spi/tracing/TracerHandle.java rename to presto-spi/src/main/java/com/facebook/presto/spi/telemetry/TelemetryFactory.java index bf9104e1545ec..43f54fbe073ed 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/TracerHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/telemetry/TelemetryFactory.java @@ -11,9 +11,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.spi.tracing; +package com.facebook.presto.spi.telemetry; -public interface TracerHandle +/** + * The interface Telemetry factory. + * + * @param the type parameter + */ +public interface TelemetryFactory { - String getTraceToken(); + /** + * Gets name. + * + * @return the name + */ + String getName(); + + /** + * Create t. + * + * @return the t + */ + T create(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/telemetry/TelemetryTracing.java b/presto-spi/src/main/java/com/facebook/presto/spi/telemetry/TelemetryTracing.java new file mode 100644 index 0000000000000..998068811c066 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/telemetry/TelemetryTracing.java @@ -0,0 +1,207 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.telemetry; + +import com.facebook.presto.common.ErrorCode; + +import java.util.Map; +import java.util.Optional; + +/** + * The interface Telemetry tracing. + * + * @param the type parameter + * @param the type parameter + */ +public interface TelemetryTracing +{ + /** + * Load configured open telemetry. + */ + void loadConfiguredOpenTelemetry(); + + /** + * Gets current context wrap. + * + * @param runnable the runnable + * @return the current context wrap + */ + Runnable getCurrentContextWrap(Runnable runnable); + + /** + * Is recording boolean. + * + * @return the boolean + */ + boolean isRecording(); + + /** + * Gets headers map. + * + * @param span the span + * @return the headers map + */ + Map getHeadersMap(T span); + + /** + * End span on error. + * + * @param querySpan the query span + * @param throwable the throwable + */ + void endSpanOnError(T querySpan, Throwable throwable); + + /** + * Add event. + * + * @param span the span + * @param eventName the event name + */ + void addEvent(T span, String eventName); + + /** + * Add event. + * + * @param querySpan the query span + * @param eventName the event name + * @param eventState the event state + */ + void addEvent(T querySpan, String eventName, String eventState); + + /** + * Sets attributes. + * + * @param span the span + * @param attributes the attributes + */ + void setAttributes(T span, Map attributes); + + /** + * Record exception. + * + * @param querySpan the query span + * @param message the message + * @param runtimeException the runtime exception + * @param errorCode the error code + */ + void recordException(T querySpan, String message, RuntimeException runtimeException, ErrorCode errorCode); + + /** + * Sets success. + * + * @param querySpan the query span + */ + void setSuccess(T querySpan); + + /** + * Gets invalid span. + * + * @return the invalid span + */ +//GetSpans + T getInvalidSpan(); + + /** + * Gets root span. + * + * @return the root span + */ +//GetSpans + T getRootSpan(); + + /** + * Gets span. + * + * @param spanName the span name + * @return the span + */ + T getSpan(String spanName); + + /** + * Gets span. + * + * @param traceParent the trace parent + * @param spanName the span name + * @return the span + */ + T getSpan(String traceParent, String spanName); + + /** + * Gets span. + * + * @param parentSpan the parent span + * @param spanName the span name + * @param attributes the attributes + * @return the span + */ + T getSpan(T parentSpan, String spanName, Map attributes); + + /** + * Span string optional. + * + * @param span the span + * @return the optional + */ + Optional spanString(T span); + + /** + * Scoped span u. + * + * @param name the name + * @param skipSpan the skip span + * @return the u + */ +//scoped spans + U scopedSpan(String name, Boolean... skipSpan); + + /** + * Scoped span u. + * + * @param span the span + * @param skipSpan the skip span + * @return the u + */ + U scopedSpan(T span, Boolean... skipSpan); + + /** + * Scoped span u. + * + * @param parentSpan the parent span + * @param spanName the span name + * @param attributes the attributes + * @param skipSpan the skip span + * @return the u + */ + U scopedSpan(T parentSpan, String spanName, Map attributes, Boolean... skipSpan); + + /** + * Scoped span u. + * + * @param parentSpan the parent span + * @param spanName the span name + * @param skipSpan the skip span + * @return the u + */ + U scopedSpan(T parentSpan, String spanName, Boolean... skipSpan); + + /** + * Scoped span u. + * + * @param spanName the span name + * @param attributes the attributes + * @param skipSpan the skip span + * @return the u + */ + U scopedSpan(String spanName, Map attributes, Boolean... skipSpan); +} diff --git a/presto-main/src/main/java/com/facebook/presto/tracing/NoopTracerHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/testing/TestingTelemetryTracing.java similarity index 61% rename from presto-main/src/main/java/com/facebook/presto/tracing/NoopTracerHandle.java rename to presto-spi/src/main/java/com/facebook/presto/spi/testing/TestingTelemetryTracing.java index a182bf535b419..bacb83ac1d33c 100644 --- a/presto-main/src/main/java/com/facebook/presto/tracing/NoopTracerHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/testing/TestingTelemetryTracing.java @@ -11,23 +11,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.tracing; +package com.facebook.presto.spi.testing; -import com.facebook.presto.spi.tracing.TracerHandle; +import java.util.List; -public class NoopTracerHandle - implements TracerHandle +public interface TestingTelemetryTracing { - private final String traceToken; + void loadConfiguredOpenTelemetry(); - public NoopTracerHandle() - { - this.traceToken = "noop_dummy_id"; - } + List getFinishedSpanItems(); - @Override - public String getTraceToken() - { - return traceToken; - } + boolean isSpansEmpty(); + + boolean spansAnyMatch(String spanName); + + void clearSpanList(); + + //OpenTelemetry getOpenTelemetry(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/NoopTracer.java b/presto-spi/src/main/java/com/facebook/presto/spi/tracing/NoopTracer.java deleted file mode 100644 index 176d62bb166c6..0000000000000 --- a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/NoopTracer.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.spi.tracing; - -public class NoopTracer - implements Tracer -{ - @Override - public void addPoint(String annotation) - { - } - - @Override - public void startBlock(String blockName, String annotation) - { - } - - @Override - public void addPointToBlock(String blockName, String annotation) - { - } - - @Override - public void endBlock(String blockName, String annotation) - { - } - - @Override - public void endTrace(String annotation) - { - } - - @Override - public String getTracerId() - { - return "noop_dummy_id"; - } -} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/Tracer.java b/presto-spi/src/main/java/com/facebook/presto/spi/tracing/Tracer.java deleted file mode 100644 index 645d7a739c576..0000000000000 --- a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/Tracer.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.spi.tracing; - -public interface Tracer -{ - String tracerName = "com.facebook.presto"; - /** - * Add a single trace point with current time to the main block - * @param annotation message associated with the trace point - */ - void addPoint(String annotation); - - /** - * Add a new block starting at current time to the trace - * @param blockName the name for the added tracing block - * @param annotation message associated with the start point of the block - */ - void startBlock(String blockName, String annotation); - - /** - * Add a trace point with current time to the specified block - * @param blockName the name for the tracing block to add point to - * @param annotation message associated with the added point to the block - */ - void addPointToBlock(String blockName, String annotation); - - /** - * End the specified block - * @param blockName the name for the tracing block to end - * @param annotation message associated with the end point of the block - */ - void endBlock(String blockName, String annotation); - - /** - * End the entire trace - * @param annotation message associated with the end point of the trace - */ - void endTrace(String annotation); - - String getTracerId(); -} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/TracerProvider.java b/presto-spi/src/main/java/com/facebook/presto/spi/tracing/TracerProvider.java deleted file mode 100644 index a7fd4988cdfcd..0000000000000 --- a/presto-spi/src/main/java/com/facebook/presto/spi/tracing/TracerProvider.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.spi.tracing; - -import java.util.Map; -import java.util.function.Function; - -public interface TracerProvider -{ - String getName(); - - /** - * Return tracer type provided by TracerProvider - */ - String getTracerType(); - - /** - * The method returns a function to take a set of HTTP headers and generate the tracer handle - */ - Function, TracerHandle> getHandleGenerator(); - - /** - * - * @return A @Tracer that should be kept throughout the whole duration of tracing. - */ - Tracer getNewTracer(TracerHandle handle); -} diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/TestingSession.java b/presto-spi/src/test/java/com/facebook/presto/spi/TestingSession.java index cc085121b6851..a38aa0312f0f1 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/TestingSession.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/TestingSession.java @@ -83,12 +83,6 @@ public long getStartTime() return 0; } - @Override - public Optional getTraceToken() - { - return Optional.empty(); - } - @Override public SqlFunctionProperties getSqlFunctionProperties() { diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index fa38be6ffb751..24ac76b2a9571 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -344,7 +344,13 @@ runtime true - + + + com.facebook.presto + presto-open-telemetry + test + + diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index 080c530875095..ac9d481d2f29f 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -3066,13 +3066,14 @@ public void testShowSession() { Session session = new Session( getSession().getQueryId(), + null, + null, Optional.empty(), getSession().isClientTransactionSupport(), getSession().getIdentity(), getSession().getSource(), getSession().getCatalog(), getSession().getSchema(), - getSession().getTraceToken(), getSession().getTimeZoneKey(), getSession().getLocale(), getSession().getRemoteUserAddress(), @@ -3093,7 +3094,6 @@ public void testShowSession() getQueryRunner().getMetadata().getSessionPropertyManager(), getSession().getPreparedStatements(), ImmutableMap.of(), - getSession().getTracer(), getSession().getWarningCollector(), getSession().getRuntimeStats(), getSession().getQueryType()); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java index 5316b636b30e0..35431ae6a6c41 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestingPrestoClient.java @@ -164,7 +164,6 @@ private static ClientSession toClientSession(Session session, URI server, Durati server, session.getIdentity().getUser(), session.getSource().orElse(null), - session.getTraceToken(), session.getClientTags(), session.getClientInfo().orElse(null), session.getCatalog().orElse(null), diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java index 9058d480a9360..ab0a80a941a20 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestFinalQueryInfo.java @@ -27,7 +27,6 @@ import org.testng.annotations.Test; import java.util.Locale; -import java.util.Optional; import static com.facebook.airlift.concurrent.MoreFutures.tryGetFutureValue; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; @@ -66,7 +65,6 @@ private static QueryId startQuery(String sql, DistributedQueryRunner queryRunner queryRunner.getCoordinator().getBaseUrl(), "user", "source", - Optional.empty(), ImmutableSet.of(), null, null, diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java index 9b748b4948c42..48103301b2b5e 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestQueryRunnerUtil.java @@ -42,7 +42,7 @@ public static QueryId createQuery(DistributedQueryRunner queryRunner, Session se public static QueryId createQuery(DistributedQueryRunner queryRunner, int coordinator, Session session, String sql) { DispatchManager dispatchManager = queryRunner.getCoordinator(coordinator).getDispatchManager(); - getFutureValue(dispatchManager.createQuery(session.getQueryId(), "slug", 0, new TestingSessionContext(session), sql)); + getFutureValue(dispatchManager.createQuery(session.getQueryId(), null, null, "slug", 0, new TestingSessionContext(session), sql)); return session.getQueryId(); } diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java index 5f6748d54c4f4..ce062a4c955ac 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestingSessionContext.java @@ -22,7 +22,6 @@ import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.session.ResourceEstimates; -import com.facebook.presto.spi.tracing.Tracer; import com.google.common.collect.ImmutableMap; import java.util.Map; @@ -66,12 +65,6 @@ public String getSource() return session.getSource().orElse(null); } - @Override - public Optional getTraceToken() - { - return session.getTraceToken(); - } - @Override public String getRemoteUserAddress() { @@ -114,12 +107,6 @@ public String getLanguage() return session.getLocale().getLanguage(); } - @Override - public Optional getTracer() - { - return session.getTracer(); - } - @Override public Map getSystemProperties() { diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java index 98739db46c920..03a9e1bd83bf4 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java @@ -14,11 +14,14 @@ package com.facebook.presto.tests; import com.facebook.presto.Session; +import com.facebook.presto.common.TelemetryConfig; import com.facebook.presto.connector.MockConnectorFactory; import com.facebook.presto.dispatcher.DispatchManager; import com.facebook.presto.execution.TestingSessionContext; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.plugin.opentelemetry.OpenTelemetryPlugin; import com.facebook.presto.server.BasicQueryInfo; +import com.facebook.presto.server.testing.TestingPrestoServer; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorTableHandle; import com.facebook.presto.spi.Plugin; @@ -26,6 +29,7 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.testing.TestingTracingManager; import com.facebook.presto.testing.TestingTransactionHandle; import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; import com.facebook.presto.transaction.TransactionBuilder; @@ -36,7 +40,9 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; @@ -46,6 +52,8 @@ import static com.facebook.presto.spi.Constraint.alwaysTrue; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; /** @@ -59,6 +67,7 @@ public class TestMetadataManager { private DistributedQueryRunner queryRunner; private MetadataManager metadataManager; + private TestingTracingManager testingTracingManager; @BeforeClass public void setUp() @@ -98,6 +107,30 @@ public void tearDown() metadataManager = null; } + @BeforeClass + public void setup() throws Exception + { + Map properties = new HashMap<>(); + properties.put("tracing-enabled", "true"); + properties.put("tracing-backend-url", "http://localhost:4317"); + properties.put("max-exporter-batch-size", "256"); + properties.put("max-queue-size", "1024"); + properties.put("exporter-timeout", "5000"); + properties.put("schedule-delay", "1000"); + properties.put("trace-sampling-ratio", "1.0"); + properties.put("span-sampling", "true"); + TelemetryConfig.getTelemetryConfig().setTelemetryProperties(properties); + + TestingPrestoServer testingPrestoServer = new TestingPrestoServer( + ImmutableMap.builder() + .put("plugin.bundles", "../presto-open-telemetry/pom.xml") + .build()); + testingPrestoServer.installPlugin(new OpenTelemetryPlugin()); + + testingTracingManager = testingPrestoServer.getTestingTracingManager(); + testingTracingManager.loadConfiguredOpenTelemetry(); + } + @Test public void testMetadataIsClearedAfterQueryFinished() { @@ -130,6 +163,8 @@ public void testMetadataIsClearedAfterQueryCanceled() QueryId queryId = dispatchManager.createQueryId(); dispatchManager.createQuery( queryId, + null, + null, "slug", 0, new TestingSessionContext(TEST_SESSION), @@ -198,4 +233,87 @@ public void testGetTableStatisticsThrows() assertEquals(queryRunner.getMetadata().getTableStatistics(transactionSession, tableHandle, ImmutableList.of(), alwaysTrue()), TableStatistics.empty()); }); } + + @Test + public void testMetadataGetCatalogsByQueryIdTraceEnabledWithSampling() throws InterruptedException + { + //With sampling + TelemetryConfig.getTelemetryConfig().setTracingEnabled(true); + TelemetryConfig.getTelemetryConfig().setSpanSampling(true); + + @Language("SQL") String sql = "SELECT * FROM nation"; + queryRunner.execute(sql); + metadataManager.getCatalogsByQueryId(); + + Thread.sleep(5000); + + assertFalse(testingTracingManager.isSpansEmpty()); + assertTrue(testingTracingManager.spansAnyMatch("query")); + assertTrue(testingTracingManager.spansAnyMatch("dispatch")); + assertFalse(testingTracingManager.spansAnyMatch("getCatalogsByQueryId")); + + testingTracingManager.clearSpanList(); + } + + @Test + public void testMetadataGetCatalogsByQueryIdTraceEnabledWithoutSampling() throws InterruptedException + { + //Without sampling + TelemetryConfig.getTelemetryConfig().setTracingEnabled(true); + TelemetryConfig.getTelemetryConfig().setSpanSampling(false); + + @Language("SQL") String sql = "SELECT * FROM nation"; + queryRunner.execute(sql); + metadataManager.getCatalogsByQueryId(); + + Thread.sleep(5000); + + assertFalse(testingTracingManager.isSpansEmpty()); + assertTrue(testingTracingManager.spansAnyMatch("query")); + assertTrue(testingTracingManager.spansAnyMatch("dispatch")); + assertTrue(testingTracingManager.spansAnyMatch("getCatalogsByQueryId")); + + testingTracingManager.clearSpanList(); + } + + @Test + public void testMetadataGetCatalogsByQueryIdTraceDisabled() throws InterruptedException + { + TelemetryConfig.getTelemetryConfig().setTracingEnabled(false); + TelemetryConfig.getTelemetryConfig().setSpanSampling(false); + + @Language("SQL") String sql = "SELECT * FROM nation"; + queryRunner.execute(sql); + metadataManager.getCatalogsByQueryId(); + + Thread.sleep(5000); + + assertTrue(testingTracingManager.isSpansEmpty()); + + testingTracingManager.clearSpanList(); + } + + @Test + public void testMetadataGetCatalogsByQueryIdTraceEnabledFailed() throws InterruptedException + { + TelemetryConfig.getTelemetryConfig().setTracingEnabled(true); + TelemetryConfig.getTelemetryConfig().setSpanSampling(true); + + @Language("SQL") String sql = "SELECT * FROM dummy"; + try { + queryRunner.execute(sql); + } + catch (Exception e) { + metadataManager.getCatalogsByQueryId(); + //testing failed query + } + + Thread.sleep(5000); + + assertFalse(testingTracingManager.isSpansEmpty()); + assertTrue(testingTracingManager.spansAnyMatch("query")); + assertTrue(testingTracingManager.spansAnyMatch("dispatch")); + + testingTracingManager.clearSpanList(); + } } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java index 7e82c52e51bf2..8b4d2c816b1fa 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryManager.java @@ -15,6 +15,7 @@ import com.facebook.presto.Session; import com.facebook.presto.common.RuntimeStats; +import com.facebook.presto.common.TelemetryConfig; import com.facebook.presto.cost.StatsAndCosts; import com.facebook.presto.dispatcher.DispatchManager; import com.facebook.presto.execution.MockQueryExecution; @@ -27,6 +28,7 @@ import com.facebook.presto.execution.TestEventListenerPlugin.TestingEventListenerPlugin; import com.facebook.presto.execution.TestingSessionContext; import com.facebook.presto.plugin.blackhole.BlackHolePlugin; +import com.facebook.presto.plugin.opentelemetry.OpenTelemetryPlugin; import com.facebook.presto.resourceGroups.FileResourceGroupConfigurationManagerFactory; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.testing.TestingPrestoServer; @@ -36,6 +38,7 @@ import com.facebook.presto.spi.WarningCode; import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; import com.facebook.presto.spi.memory.MemoryPoolId; +import com.facebook.presto.testing.TestingTracingManager; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -51,7 +54,9 @@ import java.net.URI; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; @@ -73,6 +78,7 @@ import static com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder.builder; import static com.facebook.presto.utils.ResourceUtils.getResourceFilePath; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -82,11 +88,32 @@ public class TestQueryManager { private DistributedQueryRunner queryRunner; private static final String LONG_LASTING_QUERY = "SELECT COUNT(*) FROM blackhole.default.dummy"; + private TestingTracingManager testingTracingManager; @BeforeClass public void setUp() throws Exception { + Map properties = new HashMap<>(); + properties.put("tracing-enabled", "true"); + properties.put("tracing-backend-url", "http://localhost:4317"); + properties.put("max-exporter-batch-size", "256"); + properties.put("max-queue-size", "1024"); + properties.put("exporter-timeout", "5000"); + properties.put("schedule-delay", "1000"); + properties.put("trace-sampling-ratio", "1.0"); + properties.put("span-sampling", "true"); + TelemetryConfig.getTelemetryConfig().setTelemetryProperties(properties); + + TestingPrestoServer testingPrestoServer = new TestingPrestoServer( + ImmutableMap.builder() + .put("plugin.bundles", "../presto-open-telemetry/pom.xml") + .build()); + testingPrestoServer.installPlugin(new OpenTelemetryPlugin()); + + testingTracingManager = testingPrestoServer.getTestingTracingManager(); + testingTracingManager.loadConfiguredOpenTelemetry(); + queryRunner = getSimpleQueryRunner(); queryRunner.installPlugin(new BlackHolePlugin()); queryRunner.createCatalog("blackhole", "blackhole"); @@ -119,6 +146,8 @@ public void testFailQuery() QueryId queryId = dispatchManager.createQueryId(); dispatchManager.createQuery( queryId, + null, + null, "slug", 0, new TestingSessionContext(TEST_SESSION), @@ -148,6 +177,56 @@ public void testFailQuery() assertEquals(queryManager.getStats().getQueuedQueries(), 0); } + @Test + public void testDispatchManagerCreateQueryWithTracingEnabled() throws Exception + { + TelemetryConfig.getTelemetryConfig().setTracingEnabled(true); + + DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager(); + QueryId queryId = dispatchManager.createQueryId(); + dispatchManager.createQuery( + queryId, + null, + null, + "slug", + 0, + new TestingSessionContext(TEST_SESSION), + "SELECT * FROM lineitem") + .get(); + + Thread.sleep(5000); + + assertFalse(testingTracingManager.isSpansEmpty()); + assertTrue(testingTracingManager.spansAnyMatch("dispatch")); + + testingTracingManager.clearSpanList(); + } + + @Test + public void testDispatchManagerCreateQueryWithTracingDisabled() throws Exception + { + TelemetryConfig.getTelemetryConfig().setTracingEnabled(false); + + DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager(); + QueryId queryId = dispatchManager.createQueryId(); + dispatchManager.createQuery( + queryId, + null, + null, + "slug", + 0, + new TestingSessionContext(TEST_SESSION), + "SELECT * FROM lineitem") + .get(); + + Thread.sleep(5000); + + assertFalse(testingTracingManager.isSpansEmpty()); + assertTrue(testingTracingManager.spansAnyMatch("dispatch")); + + testingTracingManager.clearSpanList(); + } + @Test(timeOut = 60_000L) public void testFailQueryPrerun() throws Exception @@ -164,6 +243,8 @@ public void testFailQueryPrerun() } dispatchManager.createQuery( queryId, + null, + null, "slug", 0, new TestingSessionContext(TEST_SESSION), @@ -201,6 +282,8 @@ void createQueries(DispatchManager dispatchManager, int queryCount) for (int i = 0; i < queryCount; i++) { dispatchManager.createQuery( dispatchManager.createQueryId(), + null, + null, "slug", 0, new TestingSessionContext(TEST_SESSION),