Compare commits
No commits in common. "34f0ac9460351552b259b8e01bdd8d736d54ddeb" and "eec81afaab2ae6701f6a54c3ed74a0a20fd1452e" have entirely different histories.
34f0ac9460
...
eec81afaab
150
lynx/convert_xla2chrome_trace.py
Normal file
150
lynx/convert_xla2chrome_trace.py
Normal file
@ -0,0 +1,150 @@
|
||||
# coding=utf-8
|
||||
from __future__ import unicode_literals, absolute_import
|
||||
import json
|
||||
from argparse import ArgumentParser
|
||||
from collections import defaultdict
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('req_json', type=str,help="reorder request file")
|
||||
parser.add_argument('res_json', type=str,help="reorder result file")
|
||||
parser.add_argument('trace_dst_file',type=str,help="write chrome trace file path")
|
||||
|
||||
args = parser.parse_args()
|
||||
with open(args.req_json) as fin:
|
||||
req_json = json.load(fin)
|
||||
with open(args.res_json) as fin:
|
||||
res_json = json.load(fin)
|
||||
uuid2node ={}
|
||||
nodes = req_json["nodes"]
|
||||
for node in nodes:
|
||||
uuid2node[node["uuid"]] = node
|
||||
# cat,dur,ts,
|
||||
edge_map = {}
|
||||
edges = req_json["edges"]
|
||||
for edge in edges:
|
||||
from_id = edge["from"]
|
||||
to_id = edge["to"]
|
||||
if uuid2node[from_id]["typename"]!="compute" and \
|
||||
uuid2node[to_id]["typename"]!="compute": # comm start node
|
||||
edge_map[from_id] = edge
|
||||
traceEvents = []
|
||||
chrome_trace_json_format = {
|
||||
"distributedInfo":{
|
||||
"rank":0
|
||||
},
|
||||
|
||||
}
|
||||
stackFrames = {}
|
||||
# antopt format
|
||||
if "finalized_node_start_end_time" in res_json:
|
||||
finalized_node_start_end_time = res_json["finalized_node_start_end_time"]
|
||||
# prepare comm edge
|
||||
for node_start_end in finalized_node_start_end_time:
|
||||
node_uuid = node_start_end["node_uuid"]
|
||||
if "_" in node_uuid:#TODO: edge,
|
||||
continue
|
||||
node = uuid2node[node_uuid]
|
||||
if node["typename"]=="compute":
|
||||
traceEvents.append({
|
||||
"id":node_uuid,
|
||||
"cat": "kernel",
|
||||
"dur": node_start_end["end_time"]-node_start_end["start_time"],
|
||||
"ts": node_start_end["start_time"],
|
||||
"pid": 0,
|
||||
"tid": 0,
|
||||
"name": node["name"],
|
||||
"ph": "X",
|
||||
"args": {
|
||||
"cost":node["cost"],
|
||||
"uuid":node_uuid
|
||||
|
||||
}
|
||||
})
|
||||
else:
|
||||
name = "ncclKernel_"+node["name"]
|
||||
if node_uuid not in edge_map:
|
||||
continue
|
||||
cost = edge_map[node_uuid]["cost"]
|
||||
traceEvents.append({
|
||||
"id":node_uuid,
|
||||
"cat": "kernel",
|
||||
"dur": cost,
|
||||
"ts": node_start_end["start_time"],
|
||||
"pid": 0,
|
||||
"tid": 1,
|
||||
"name": name,
|
||||
"ph": "X",
|
||||
"args": {
|
||||
"cost": cost,
|
||||
"uuid":node_uuid
|
||||
|
||||
}
|
||||
})
|
||||
elif "nodes" in res_json:
|
||||
# ortools format
|
||||
# prepare communication edge
|
||||
node2deps = defaultdict(list)
|
||||
for edge in req_json["edges"]:
|
||||
node2deps[edge["to"]].append(edge["from"])
|
||||
for node in res_json["nodes"]:
|
||||
node_before_solve = uuid2node[node["uuid"]]
|
||||
# overwrite
|
||||
node["opcode"] = node_before_solve["opcode"]
|
||||
uuid2node[node["uuid"]]=node
|
||||
def normalize_dt(time):
|
||||
return time
|
||||
for node in res_json["nodes"]:
|
||||
uuid = node["uuid"]
|
||||
deps = node2deps[uuid]
|
||||
|
||||
if node["typename"]=="compute":
|
||||
traceEvents.append({
|
||||
"id": uuid,
|
||||
"cat": "kernel",
|
||||
"dur": normalize_dt(node["endTime"]-node["startTime"]),
|
||||
"ts": normalize_dt(node["startTime"]),
|
||||
"pid": 0,
|
||||
"tid": 0,
|
||||
"name": node["name"],
|
||||
"ph":"X",
|
||||
"args":{
|
||||
"cost": normalize_dt(node["endTime"]-node["startTime"]),
|
||||
"uuid":uuid,
|
||||
"opcode":node["opcode"],
|
||||
"obj":{
|
||||
"id_ref":node2deps[uuid]
|
||||
},
|
||||
},
|
||||
})
|
||||
else:
|
||||
name = "ncclKernel_"+node["name"]
|
||||
if uuid in edge_map:
|
||||
cost = edge_map[uuid]["cost"]
|
||||
else:
|
||||
# compute link to communication
|
||||
continue
|
||||
traceEvents.append({
|
||||
"id":uuid,
|
||||
"cat": "kernel",
|
||||
"dur":normalize_dt(cost),
|
||||
"ts": normalize_dt(node["startTime"]),
|
||||
"pid": 0,
|
||||
"tid": 1,
|
||||
"name": name,
|
||||
"ph": "X",
|
||||
"args": {
|
||||
"cost": normalize_dt(cost),
|
||||
"uuid":uuid,
|
||||
"opcode":node["opcode"]
|
||||
|
||||
}
|
||||
})
|
||||
count = 0
|
||||
|
||||
chrome_trace_json_format["traceEvents"] = traceEvents
|
||||
chrome_trace_json_format["stackFrames"] = stackFrames
|
||||
with open(args.trace_dst_file, "w") as fout:
|
||||
json.dump(chrome_trace_json_format, fout,indent=2)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
44
uv.lock
generated
44
uv.lock
generated
@ -407,20 +407,40 @@ requires-dist = [
|
||||
|
||||
[[package]]
|
||||
name = "numpy"
|
||||
version = "2.0.2"
|
||||
version = "2.2.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fb/90/8956572f5c4ae52201fdec7ba2044b2c882832dcec7d5d0922c9e9acf2de/numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020", size = 20262700 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803 },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835 },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497 },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158 },
|
||||
{ url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174 },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313 },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179 },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/ec/43628dcf98466e087812142eec6d1c1a6c6bdfdad30a0aa07b872dc01f6f/numpy-2.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d", size = 20929458 },
|
||||
{ url = "https://files.pythonhosted.org/packages/9b/c0/2f4225073e99a5c12350954949ed19b5d4a738f541d33e6f7439e33e98e4/numpy-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95", size = 14115299 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/fa/d2c5575d9c734a7376cc1592fae50257ec95d061b27ee3dbdb0b3b551eb2/numpy-2.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea", size = 5145723 },
|
||||
{ url = "https://files.pythonhosted.org/packages/eb/dc/023dad5b268a7895e58e791f28dc1c60eb7b6c06fcbc2af8538ad069d5f3/numpy-2.2.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532", size = 6678797 },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/19/bcd641ccf19ac25abb6fb1dcd7744840c11f9d62519d7057b6ab2096eb60/numpy-2.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e", size = 14067362 },
|
||||
{ url = "https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe", size = 16116679 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/a1/e90f7aa66512be3150cb9d27f3d9995db330ad1b2046474a13b7040dfd92/numpy-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021", size = 15264272 },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/b6/50bd027cca494de4fa1fc7bf1662983d0ba5f256fa0ece2c376b5eb9b3f0/numpy-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8", size = 17880549 },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/30/f7bf4acb5f8db10a96f73896bdeed7a63373137b131ca18bd3dab889db3b/numpy-2.2.3-cp312-cp312-win32.whl", hash = "sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe", size = 6293394 },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d", size = 12626357 },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/8b/88b98ed534d6a03ba8cddb316950fe80842885709b58501233c29dfa24a9/numpy-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bfdb06b395385ea9b91bf55c1adf1b297c9fdb531552845ff1d3ea6e40d5aba", size = 20916001 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/b4/def6ec32c725cc5fbd8bdf8af80f616acf075fe752d8a23e895da8c67b70/numpy-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:23c9f4edbf4c065fddb10a4f6e8b6a244342d95966a48820c614891e5059bb50", size = 14130721 },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/60/70af0acc86495b25b672d403e12cb25448d79a2b9658f4fc45e845c397a8/numpy-2.2.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:a0c03b6be48aaf92525cccf393265e02773be8fd9551a2f9adbe7db1fa2b60f1", size = 5130999 },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/69/d96c006fb73c9a47bcb3611417cf178049aae159afae47c48bd66df9c536/numpy-2.2.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:2376e317111daa0a6739e50f7ee2a6353f768489102308b0d98fcf4a04f7f3b5", size = 6665299 },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/3f/d8a877b6e48103733ac224ffa26b30887dc9944ff95dffdfa6c4ce3d7df3/numpy-2.2.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fb62fe3d206d72fe1cfe31c4a1106ad2b136fcc1606093aeab314f02930fdf2", size = 14064096 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/43/619c2c7a0665aafc80efca465ddb1f260287266bdbdce517396f2f145d49/numpy-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52659ad2534427dffcc36aac76bebdd02b67e3b7a619ac67543bc9bfe6b7cdb1", size = 16114758 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/79/ee4fe4f60967ccd3897aa71ae14cdee9e3c097e3256975cc9575d393cb42/numpy-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b416af7d0ed3271cad0f0a0d0bee0911ed7eba23e66f8424d9f3dfcdcae1304", size = 15259880 },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/c8/8b55cf05db6d85b7a7d414b3d1bd5a740706df00bfa0824a08bf041e52ee/numpy-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1402da8e0f435991983d0a9708b779f95a8c98c6b18a171b9f1be09005e64d9d", size = 17876721 },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/d6/b4c2f0564b7dcc413117b0ffbb818d837e4b29996b9234e38b2025ed24e7/numpy-2.2.3-cp313-cp313-win32.whl", hash = "sha256:136553f123ee2951bfcfbc264acd34a2fc2f29d7cdf610ce7daf672b6fbaa693", size = 6290195 },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/e7/7d55a86719d0de7a6a597949f3febefb1009435b79ba510ff32f05a8c1d7/numpy-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:5b732c8beef1d7bc2d9e476dbba20aaff6167bf205ad9aa8d30913859e82884b", size = 12619013 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/1f/0b863d5528b9048fd486a56e0b97c18bf705e88736c8cea7239012119a54/numpy-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:435e7a933b9fda8126130b046975a968cc2d833b505475e588339e09f7672890", size = 20944621 },
|
||||
{ url = "https://files.pythonhosted.org/packages/aa/99/b478c384f7a0a2e0736177aafc97dc9152fc036a3fdb13f5a3ab225f1494/numpy-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7678556eeb0152cbd1522b684dcd215250885993dd00adb93679ec3c0e6e091c", size = 14142502 },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/61/2d9a694a0f9cd0a839501d362de2a18de75e3004576a3008e56bdd60fcdb/numpy-2.2.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2e8da03bd561504d9b20e7a12340870dfc206c64ea59b4cfee9fceb95070ee94", size = 5176293 },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/35/51e94011b23e753fa33f891f601e5c1c9a3d515448659b06df9d40c0aa6e/numpy-2.2.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:c9aa4496fd0e17e3843399f533d62857cef5900facf93e735ef65aa4bbc90ef0", size = 6691874 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ff/cf/06e37619aad98a9d03bd8d65b8e3041c3a639be0f5f6b0a0e2da544538d4/numpy-2.2.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4ca91d61a4bf61b0f2228f24bbfa6a9facd5f8af03759fe2a655c50ae2c6610", size = 14036826 },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/93/5d7d19955abd4d6099ef4a8ee006f9ce258166c38af259f9e5558a172e3e/numpy-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deaa09cd492e24fd9b15296844c0ad1b3c976da7907e1c1ed3a0ad21dded6f76", size = 16096567 },
|
||||
{ url = "https://files.pythonhosted.org/packages/af/53/d1c599acf7732d81f46a93621dab6aa8daad914b502a7a115b3f17288ab2/numpy-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:246535e2f7496b7ac85deffe932896a3577be7af8fb7eebe7146444680297e9a", size = 15242514 },
|
||||
{ url = "https://files.pythonhosted.org/packages/53/43/c0f5411c7b3ea90adf341d05ace762dad8cb9819ef26093e27b15dd121ac/numpy-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:daf43a3d1ea699402c5a850e5313680ac355b4adc9770cd5cfc2940e7861f1bf", size = 17872920 },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/57/6dbdd45ab277aff62021cafa1e15f9644a52f5b5fc840bc7591b4079fb58/numpy-2.2.3-cp313-cp313t-win32.whl", hash = "sha256:cf802eef1f0134afb81fef94020351be4fe1d6681aadf9c5e862af6602af64ef", size = 6346584 },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/9b/484f7d04b537d0a1202a5ba81c6f53f1846ae6c63c2127f8df869ed31342/numpy-2.2.3-cp313-cp313t-win_amd64.whl", hash = "sha256:aee2512827ceb6d7f517c8b85aa5d3923afe8fc7a57d028cffcd522f1c6fd082", size = 12706784 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user