Skip to content

Commit

Permalink
Bringing new query profile collector fields with concurrent search ex…
Browse files Browse the repository at this point in the history 
…ecution

Signed-off-by: Ticheng Lin <ticheng@amazon.com>
  • Loading branch information
@ticheng-aws
ticheng-aws committed Jun 2, 2023
1 parent 274866b commit 8269e42
Show file tree
Hide file tree
Showing 8 changed files with 326 additions and 19 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Changed
- Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
- Add min, max, average and thread info to resource stats in tasks API ([#7673](https://github.com/opensearch-project/OpenSearch/pull/7673))
- Add new query profile collector fields with concurrent search execution ([#7355](https://github.com/opensearch-project/OpenSearch/pull/7355))

### Deprecated

Expand Down
126 changes: 121 additions & 5 deletions server/src/main/java/org/opensearch/search/profile/query/CollectorResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@ public class CollectorResult implements ToXContentObject, Writeable {
private static final ParseField REASON = new ParseField("reason");
private static final ParseField TIME = new ParseField("time");
private static final ParseField TIME_NANOS = new ParseField("time_in_nanos");
private static final ParseField REDUCE_TIME_NANOS = new ParseField("reduce_time_in_nanos");
private static final ParseField MAX_SLICE_TIME_NANOS = new ParseField("max_slice_time_in_nanos");
private static final ParseField MIN_SLICE_TIME_IN_NANOS = new ParseField("min_slice_time_in_nanos");
private static final ParseField AVG_SLICE_TIME_IN_NANOS = new ParseField("avg_slice_time_in_nanos");
private static final ParseField SLICE_COUNT = new ParseField("slice_count");
private static final ParseField CHILDREN = new ParseField("children");

/**
Expand All @@ -88,6 +93,31 @@ public class CollectorResult implements ToXContentObject, Writeable {
*/
private final Long time;

/**
* The total elapsed time in reduce phase for this CollectorManager
*/
private final Long reduceTime;

/**
* The maximum slice time for this CollectorManager
*/
private final Long maxSliceTime;

/**
* The minimum slice time for this CollectorManager
*/
private final Long minSliceTime;

/**
* The average slice time for this CollectorManager
*/
private final Long avgSliceTime;

/**
* The segment slice count for this CollectorManager
*/
private final int sliceCount;

/**
* A list of children collectors "embedded" inside this collector
*/
Expand All @@ -97,6 +127,24 @@ public CollectorResult(String collectorName, String reason, Long time, List<Coll
this.collectorName = collectorName;
this.reason = reason;
this.time = time;
this.reduceTime = 0L;
this.maxSliceTime = time;
this.minSliceTime = time;
this.avgSliceTime = time;
this.sliceCount = 1;
this.children = children;
}

public CollectorResult(String collectorName, String reason, Long time, Long reduceTime, Long maxSliceTime, Long minSliceTime,
Long avgSliceTime, int sliceCount, List<CollectorResult> children) {
this.collectorName = collectorName;
this.reason = reason;
this.time = time;
this.reduceTime = reduceTime;
this.maxSliceTime = maxSliceTime;
this.minSliceTime = minSliceTime;
this.avgSliceTime = avgSliceTime;
this.sliceCount = sliceCount;
this.children = children;
}

Expand All @@ -107,6 +155,11 @@ public CollectorResult(StreamInput in) throws IOException {
this.collectorName = in.readString();
this.reason = in.readString();
this.time = in.readLong();
this.reduceTime = in.readLong();
this.maxSliceTime = in.readLong();
this.minSliceTime = in.readLong();
this.avgSliceTime = in.readLong();
this.sliceCount = in.readVInt();
int size = in.readVInt();
this.children = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
Expand All @@ -120,35 +173,75 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeString(collectorName);
out.writeString(reason);
out.writeLong(time);
out.writeLong(reduceTime);
out.writeLong(maxSliceTime);
out.writeLong(minSliceTime);
out.writeLong(avgSliceTime);
out.writeVInt(sliceCount);
out.writeVInt(children.size());
for (CollectorResult child : children) {
child.writeTo(out);
}
}

/**
* @return the profiled time for this collector (inclusive of children)
* @return the profiled time for this collector/collector manager (inclusive of children)
*/
public long getTime() {
return this.time;
}

/**
* @return a human readable "hint" about what this collector was used for
* @return the profiled reduce time for this collector manager (inclusive of children)
*/
public long getReduceTime() {
return this.reduceTime;
}

/**
* @return the profiled maximum slice time for this collector manager (inclusive of children)
*/
public long getMaxSliceTime() {
return this.maxSliceTime;
}

/**
* @return the profiled minimum slice time for this collector manager (inclusive of children)
*/
public long getMinSliceTime() {
return this.minSliceTime;
}

/**
* @return the profiled average slice time for this collector manager (inclusive of children)
*/
public long getAvgSliceTime() {
return this.avgSliceTime;
}

/**
* @return the profiled segment slice count for this collector manager (inclusive of children)
*/
public long getSliceCount() {
return this.sliceCount;
}

/**
* @return a human readable "hint" about what this collector/collector manager was used for
*/
public String getReason() {
return this.reason;
}

/**
* @return the lucene class name of the collector
* @return the lucene class name of the collector/collector manager
*/
public String getName() {
return this.collectorName;
}

/**
* @return a list of children collectors
* @return a list of children collectors/collector managers
*/
public List<CollectorResult> getProfiledChildren() {
return children;
Expand All @@ -163,6 +256,13 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
builder.field(TIME.getPreferredName(), new TimeValue(getTime(), TimeUnit.NANOSECONDS).toString());
}
builder.field(TIME_NANOS.getPreferredName(), getTime());
if (getName().contains("CollectorManager")) {
builder.field(REDUCE_TIME_NANOS.getPreferredName(), getReduceTime());
builder.field(MAX_SLICE_TIME_NANOS.getPreferredName(), getMaxSliceTime());
builder.field(MIN_SLICE_TIME_IN_NANOS.getPreferredName(), getMinSliceTime());
builder.field(AVG_SLICE_TIME_IN_NANOS.getPreferredName(), getAvgSliceTime());
builder.field(SLICE_COUNT.getPreferredName(), getSliceCount());
}

if (!children.isEmpty()) {
builder = builder.startArray(CHILDREN.getPreferredName());
Expand All @@ -181,6 +281,11 @@ public static CollectorResult fromXContent(XContentParser parser) throws IOExcep
String currentFieldName = null;
String name = null, reason = null;
long time = -1;
long reduceTime = -1;
long maxSliceTime = -1;
long minSliceTime = -1;
long avgSliceTime = -1;
int sliceCount = 0;
List<CollectorResult> children = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
Expand All @@ -195,6 +300,16 @@ public static CollectorResult fromXContent(XContentParser parser) throws IOExcep
parser.text();
} else if (TIME_NANOS.match(currentFieldName, parser.getDeprecationHandler())) {
time = parser.longValue();
} else if (REDUCE_TIME_NANOS.match(currentFieldName, parser.getDeprecationHandler())) {
reduceTime = parser.longValue();
} else if (MAX_SLICE_TIME_NANOS.match(currentFieldName, parser.getDeprecationHandler())) {
maxSliceTime = parser.longValue();
} else if (MIN_SLICE_TIME_IN_NANOS.match(currentFieldName, parser.getDeprecationHandler())) {
minSliceTime = parser.longValue();
} else if (AVG_SLICE_TIME_IN_NANOS.match(currentFieldName, parser.getDeprecationHandler())) {
avgSliceTime = parser.longValue();
} else if (SLICE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) {
sliceCount = parser.intValue();
} else {
parser.skipChildren();
}
Expand All @@ -210,6 +325,7 @@ public static CollectorResult fromXContent(XContentParser parser) throws IOExcep
parser.skipChildren();
}
}
return new CollectorResult(name, reason, time, children);
return new CollectorResult(name, reason, time, reduceTime, maxSliceTime, minSliceTime, avgSliceTime, sliceCount,
children);
}
}
76 changes: 76 additions & 0 deletions server/src/main/java/org/opensearch/search/profile/query/InternalProfileCollector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,47 @@ public class InternalProfileCollector implements Collector, InternalProfileCompo
/** The wrapped collector */
private final ProfileCollector collector;

/** The elapsed time in the concurrent search reduce phase */
private final Long reduceTime;

/** The maximum slice time for this collector manager */
private final Long maxSliceTime;

/** The minimum slice time for this collector manager */
private final Long minSliceTime;

/** The average slice time for this collector manager */
private final Long avgSliceTime;

/** The segment slice count for this collector manager */
private final int sliceCount;

/**
* A list of "embedded" children collectors
*/
private final List<? extends InternalProfileComponent> children;

public InternalProfileCollector(Collector collector, String reason, List<? extends InternalProfileComponent> children) {
this.collector = new ProfileCollector(collector);
this.reduceTime = null;
this.maxSliceTime = null;
this.minSliceTime = null;
this.avgSliceTime = null;
this.sliceCount = 1;
this.reason = reason;
this.collectorName = deriveCollectorName(collector);
this.children = children;
}

public InternalProfileCollector(Collector collector, Long reduceTime, Long maxCollectorTime, Long minSliceTime,
Long avgSliceTime, int sliceCount, String reason,
List<? extends InternalProfileComponent> children) {
this.collector = new ProfileCollector(collector);
this.reduceTime = reduceTime;
this.maxSliceTime = maxCollectorTime;
this.minSliceTime = minSliceTime;
this.avgSliceTime = avgSliceTime;
this.sliceCount = sliceCount;
this.reason = reason;
this.collectorName = deriveCollectorName(collector);
this.children = children;
Expand All @@ -87,6 +121,48 @@ public long getTime() {
return collector.getTime();
}

/**
* @return the profiled start time for this collector (inclusive of children)
*/
public long getSliceStartTime() {
return collector.getSliceStartTime();
}

/**
* @return the profiled reduce time for this collector (inclusive of children)
*/
public long getReduceTime() {
return this.reduceTime;
}

/**
* @return the profiled maximum time for this collector (inclusive of children)
*/
public long getMaxSliceTime() {
return this.maxSliceTime;
}

/**
* @return the profiled minimum time for this collector (inclusive of children)
*/
public long getMinSliceTime() {
return this.minSliceTime;
}

/**
* @return the profiled average time for this collector (inclusive of children)
*/
public long getAvgSliceTime() {
return this.avgSliceTime;
}

/**
* @return the profiled segment slice count for this collector (inclusive of children)
*/
public int getSliceCount() {
return this.sliceCount;
}

/**
* @return a human readable "hint" about what this collector was used for
*/
Expand Down
Loading

0 comments on commit 8269e42

Please sign in to comment.