UI 树日志
出于数据收集目的,UFO 可以在每个步骤保存应用程序窗口的整个 UI 树。UI 树可以表示应用程序的 UI 结构,包括窗口、控件及其属性。UI 树日志保存在 logs/{task_name}/ui_tree
文件夹中。您必须在 config_dev.yaml
文件中将 SAVE_UI_TREE
标志设置为 True
才能启用 UI 树日志。以下是应用程序 UI 树日志的示例
{
"id": "node_0",
"name": "Mail - Chaoyun Zhang - Outlook",
"control_type": "Window",
"rectangle": {
"left": 628,
"top": 258,
"right": 3508,
"bottom": 1795
},
"adjusted_rectangle": {
"left": 0,
"top": 0,
"right": 2880,
"bottom": 1537
},
"relative_rectangle": {
"left": 0.0,
"top": 0.0,
"right": 1.0,
"bottom": 1.0
},
"level": 0,
"children": [
{
"id": "node_1",
"name": "",
"control_type": "Pane",
"rectangle": {
"left": 3282,
"top": 258,
"right": 3498,
"bottom": 330
},
"adjusted_rectangle": {
"left": 2654,
"top": 0,
"right": 2870,
"bottom": 72
},
"relative_rectangle": {
"left": 0.9215277777777777,
"top": 0.0,
"right": 0.9965277777777778,
"bottom": 0.0468445022771633
},
"level": 1,
"children": []
}
]
}
UI 树日志中的字段
以下是 UI 树日志中字段的表格
字段 |
描述 |
类型 |
id |
UI 树节点的唯一标识符。 |
字符串 |
name |
UI 树节点的名称。 |
字符串 |
control_type |
UI 树节点的类型。 |
字符串 |
rectangle |
UI 树节点的绝对位置。 |
字典 |
adjusted_rectangle |
UI 树节点的调整位置。 |
字典 |
relative_rectangle |
UI 树节点的相对位置。 |
字典 |
level |
UI 树节点的级别。 |
整数 |
children |
UI 树节点的子节点。 |
UI 树节点列表 |
参考
表示 UI 树的类。
使用根元素初始化 UI 树。
源代码位于 automator/ui_control/ui_tree.py
20
21
22
23
24
25
26
27
28
29
30
31
32
33 | def __init__(self, root: UIAWrapper):
"""
Initialize the UI tree with the root element.
:param root: The root element of the UI tree.
"""
self.root = root
# The node counter to count the number of nodes in the UI tree.
self.node_counter = 0
try:
self._ui_tree = self._get_ui_tree(self.root)
except Exception as e:
self._ui_tree = {"error": traceback.format_exc()}
|
apply_ui_tree_diff(ui_tree_1, diff)
staticmethod
将 UI 树差异应用于 ui_tree_1 以获取 ui_tree_2。
参数 |
-
ui_tree_1 (Dict[str, Any] ) –
-
diff (Dict[str, Any] ) –
|
源代码位于 automator/ui_control/ui_tree.py
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323 | @staticmethod
def apply_ui_tree_diff(
ui_tree_1: Dict[str, Any], diff: Dict[str, Any]
) -> Dict[str, Any]:
"""
Apply a UI tree diff to ui_tree_1 to get ui_tree_2.
:param ui_tree_1: The original UI tree.
:param diff: The diff to apply.
:return: The new UI tree after applying the diff.
"""
ui_tree_2 = copy.deepcopy(ui_tree_1)
# Build an ID map for quick node lookups
def build_id_map(node, id_map):
id_map[node["id"]] = node
for child in node.get("children", []):
build_id_map(child, id_map)
id_map = {}
if "id" in ui_tree_2:
build_id_map(ui_tree_2, id_map)
def remove_node_by_path(path):
# The path is a list of IDs from root to target node.
# The target node is the last element. Its parent is the second to last element.
if len(path) == 1:
# Removing the root
for k in list(ui_tree_2.keys()):
del ui_tree_2[k]
id_map.clear()
return
target_id = path[-1]
parent_id = path[-2]
parent_node = id_map[parent_id]
# Find and remove the child with target_id
for i, c in enumerate(parent_node.get("children", [])):
if c["id"] == target_id:
parent_node["children"].pop(i)
break
# Remove target_id from id_map
if target_id in id_map:
del id_map[target_id]
def add_node_by_path(path, node):
# Add the node at the specified path. The parent is path[-2], the node is path[-1].
# The path[-1] should be node["id"].
if len(path) == 1:
# Replacing the root node entirely
for k in list(ui_tree_2.keys()):
del ui_tree_2[k]
for k, v in node.items():
ui_tree_2[k] = v
# Rebuild id_map
id_map.clear()
if "id" in ui_tree_2:
build_id_map(ui_tree_2, id_map)
return
target_id = path[-1]
parent_id = path[-2]
parent_node = id_map[parent_id]
# Ensure children list exists
if "children" not in parent_node:
parent_node["children"] = []
# Insert or append the node
# We don't have a numeric index anymore, we just append, assuming order doesn't matter.
# If order matters, we must store ordering info or do some heuristic.
parent_node["children"].append(node)
# Update the id_map with the newly added subtree
build_id_map(node, id_map)
def modify_node_by_path(path, changes):
# Modify fields of the node at the given ID
target_id = path[-1]
node = id_map[target_id]
for field, (old_val, new_val) in changes.items():
node[field] = new_val
# Apply removals first
# Sort removals by length of path descending so we remove deeper nodes first.
# This ensures we don't remove parents before children.
for removal in sorted(
diff["removed"], key=lambda x: len(x["path"]), reverse=True
):
remove_node_by_path(removal["path"])
# Apply additions
# Additions can be applied directly.
for addition in diff["added"]:
add_node_by_path(addition["path"], addition["node"])
# Apply modifications
for modification in diff["modified"]:
modify_node_by_path(modification["path"], modification["changes"])
return ui_tree_2
|
flatten_ui_tree()
以广度优先顺序将 UI 树展平为列表。
源代码位于 automator/ui_control/ui_tree.py
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144 | def flatten_ui_tree(self) -> List[Dict[str, Any]]:
"""
Flatten the UI tree into a list in width-first order.
"""
def flatten_tree(tree: Dict[str, Any], result: List[Dict[str, Any]]):
"""
Flatten the tree.
:param tree: The tree to flatten.
:param result: The result list.
"""
tree_info = {
"name": tree["name"],
"control_type": tree["control_type"],
"rectangle": tree["rectangle"],
"adjusted_rectangle": tree["adjusted_rectangle"],
"relative_rectangle": tree["relative_rectangle"],
"level": tree["level"],
}
result.append(tree_info)
for child in tree.get("children", []):
flatten_tree(child, result)
result = []
flatten_tree(self.ui_tree, result)
return result
|
save_ui_tree_to_json(file_path)
将 UI 树保存到 JSON 文件。
源代码位于 automator/ui_control/ui_tree.py
103
104
105
106
107
108
109
110
111
112
113
114
115 | def save_ui_tree_to_json(self, file_path: str) -> None:
"""
Save the UI tree to a JSON file.
:param file_path: The file path to save the UI tree.
"""
# Check if the file directory exists. If not, create it.
save_dir = os.path.dirname(file_path)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
with open(file_path, "w") as file:
json.dump(self.ui_tree, file, indent=4)
|
ui_tree_diff(ui_tree_1, ui_tree_2)
staticmethod
计算两个 UI 树之间的差异。
参数 |
-
ui_tree_1 (Dict[str, Any] ) –
-
ui_tree_2 (Dict[str, Any] ) –
|
源代码位于 automator/ui_control/ui_tree.py
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222 | @staticmethod
def ui_tree_diff(ui_tree_1: Dict[str, Any], ui_tree_2: Dict[str, Any]):
"""
Compute the difference between two UI trees.
:param ui_tree_1: The first UI tree.
:param ui_tree_2: The second UI tree.
:return: The difference between the two UI trees.
"""
diff = {"added": [], "removed": [], "modified": []}
def compare_nodes(node1, node2, path):
# Note: `path` is a list of IDs. The last element corresponds to the current node.
# If node1 doesn't exist and node2 does, it's an addition.
if node1 is None and node2 is not None:
diff["added"].append({"path": path, "node": copy.deepcopy(node2)})
return
# If node1 exists and node2 doesn't, it's a removal.
if node1 is not None and node2 is None:
diff["removed"].append({"path": path, "node": copy.deepcopy(node1)})
return
# If both don't exist, nothing to do.
if node1 is None and node2 is None:
return
# Both nodes exist, check for modifications at this node
fields_to_compare = [
"name",
"control_type",
"rectangle",
"adjusted_rectangle",
"relative_rectangle",
"level",
]
changes = {}
for field in fields_to_compare:
if node1[field] != node2[field]:
changes[field] = (node1[field], node2[field])
if changes:
diff["modified"].append({"path": path, "changes": changes})
# Compare children
children1 = node1.get("children", [])
children2 = node2.get("children", [])
# We'll assume children order is stable. If not, differences will appear as adds/removes.
max_len = max(len(children1), len(children2))
for i in range(max_len):
c1 = children1[i] if i < len(children1) else None
c2 = children2[i] if i < len(children2) else None
# Use the child's id if available from c2 (prefer new tree), else from c1
if c2 is not None:
child_id = c2["id"]
elif c1 is not None:
child_id = c1["id"]
else:
# Both None shouldn't happen since max_len ensures one must exist
child_id = "unknown_child_id"
compare_nodes(c1, c2, path + [child_id])
# Initialize the path with the root node id if it exists
if ui_tree_2 and "id" in ui_tree_2:
root_id = ui_tree_2["id"]
elif ui_tree_1 and "id" in ui_tree_1:
root_id = ui_tree_1["id"]
else:
# If no root id is present, assume a placeholder
root_id = "root"
compare_nodes(ui_tree_1, ui_tree_2, [root_id])
return diff
|
注意
保存 UI 树日志可能会增加系统延迟。建议在不需要 UI 树日志时将 SAVE_UI_TREE
标志设置为 False
。