Revision of compare_manifest

anduin revised this gist 5 months ago. Go to revision

1 file changed, 19 insertions, 19 deletions

compare_manifest.sh

			@@ -3,9 +3,9 @@
3	3		"""
4	4		compare_manifests.py
5	5
6		-	读取三个 filesystem.manifest 文件，比较包的并集，并输出包含 Section、Priority、Description 及各发行版存在情况的 Markdown 表格。
	6	+	读取三个 filesystem.manifest 文件，比较包的并集，并输出包含发行版勾选标记、Priority、Section、Description 的 Markdown 表格。
7	7		增加详细日志输出，并使用多线程加速 apt show 调用。
8		-	修复：自动剥离包名中的 ":架构" 后缀。
	8	+	修复：自动剥离包名中的 ":<arch>" 后缀。
9	9		"""
10	10		import sys
11	11		import subprocess
			@@ -34,8 +34,7 @@ def parse_manifest(path):
34	34		line = line.strip()
35	35		if not line or line.startswith('#'):
36	36		continue
37		-	parts = line.split()
38		-	raw_pkg = parts[0]
	37	+	raw_pkg = line.split()[0]
39	38		pkg = raw_pkg.split(':', 1)[0]
40	39		pkgs.add(pkg)
41	40		logging.info(f"Found {len(pkgs)} unique packages in {path}")
			@@ -47,7 +46,7 @@ def parse_manifest(path):
47	46
48	47		def get_pkg_info(pkg):
49	48		"""
50		-	调用 `apt show pkg`，解析 Section、Priority、Description（首段），
	49	+	调用 `apt show pkg`，解析 Priority、Section、Description（首段），
51	50		返回 dict 包含信息。
52	51		若调用失败，返回空字段。
53	52		"""
			@@ -63,14 +62,14 @@ def get_pkg_info(pkg):
63	62		)
64	63		except subprocess.TimeoutExpired:
65	64		logging.warning(f"Timeout fetching info for {pkg}")
66		-	return {"Section": "", "Priority": "", "Description": ""}
	65	+	return {"Priority": "", "Section": "", "Description": ""}
67	66
68	67		if p.returncode != 0:
69	68		logging.debug(f"apt show failed for {pkg}")
70		-	return {"Section": "", "Priority": "", "Description": ""}
	69	+	return {"Priority": "", "Section": "", "Description": ""}
71	70
72		-	section = ""
73	71		priority = ""
	72	+	section = ""
74	73		desc_lines = []
75	74		in_desc = False
76	75
			@@ -80,18 +79,18 @@ def get_pkg_info(pkg):
80	79		break
81	80		desc_lines.append(line.strip())
82	81		else:
83		-	if line.startswith("Section:"):
84		-	section = line.split(":", 1)[1].strip()
85		-	elif line.startswith("Priority:"):
	82	+	if line.startswith("Priority:"):
86	83		priority = line.split(":", 1)[1].strip()
	84	+	elif line.startswith("Section:"):
	85	+	section = line.split(":", 1)[1].strip()
87	86		elif line.startswith("Description:"):
88	87		desc = line.split(":", 1)[1].strip()
89	88		desc_lines.append(desc)
90	89		in_desc = True
91	90
92	91		full_desc = " ".join(desc_lines)
93		-	logging.debug(f"Obtained info for {pkg}: section={section}, priority={priority}")
94		-	return {"Section": section, "Priority": priority, "Description": full_desc}
	92	+	logging.debug(f"Obtained info for {pkg}: priority={priority}, section={section}")
	93	+	return {"Priority": priority, "Section": section, "Description": full_desc}
95	94
96	95
97	96		def main():
			@@ -126,22 +125,23 @@ def main():
126	125		logging.info(f"[{idx}/{total}] Processed {pkg}")
127	126		except Exception as e:
128	127		logging.error(f"Error processing {pkg}: {e}")
129		-	pkg_infos[pkg] = {"Section": "", "Priority": "", "Description": ""}
	128	+	pkg_infos[pkg] = {"Priority": "", "Section": "", "Description": ""}
130	129
131	130		# 写入 Markdown
132	131		out_path = "comp_result.md"
133	132		logging.info(f"Writing results to {out_path}")
134	133		with open(out_path, "w", encoding="utf-8") as out:
135		-	out.write("\| Package \| Section \| Priority \| Description \| Ubuntu \| Zorin \| Anduin \|\n")
136		-	out.write("\|---------\|---------\|----------\|-------------\|--------\|-------\|--------\|\n")
	134	+	# 新表头：检查标记 -> Priority -> Section -> Description
	135	+	out.write("\| Package \| Ubuntu \| Zorin \| Anduin \| Priority \| Section \| Description \|\n")
	136	+	out.write("\|---------\|--------\|-------\|--------\|----------\|---------\|-------------\|\n")
137	137
138	138		for pkg in all_pkgs:
139		-	info = pkg_infos.get(pkg, {"Section": "", "Priority": "", "Description": ""})
	139	+	info = pkg_infos.get(pkg, {"Priority": "", "Section": "", "Description": ""})
140	140		mark = lambda name: "√" if pkg in sets[name] else ""
141	141		desc = info["Description"].replace("\|", "\\\|")
142	142		out.write(
143		-	f"\| {pkg} \| {info['Section']} \| {info['Priority']} \| {desc} \|"
144		-	f" {mark('Ubuntu')} \| {mark('Zorin')} \| {mark('Anduin')} \|\n"
	143	+	f"\| {pkg} \| {mark('Ubuntu')} \| {mark('Zorin')} \| {mark('Anduin')} \|"
	144	+	f" {info['Priority']} \| {info['Section']} \| {desc} \|\n"
145	145		)
146	146
147	147		elapsed = time.time() - start_time

anduin revised this gist 5 months ago. Go to revision

No changes

anduin revised this gist 5 months ago. Go to revision

2 files changed, 159 insertions, 34 deletions

RUN.sh(file created)

		@@ -0,0 +1,8 @@
1	+	# 保存脚本为 compare_manifests.py，并赋可执行权限
2	+	chmod +x compare_manifests.py
3	+
4	+	# 运行脚本，依次传入 Ubuntu、Zorin、Anduin 三个 manifest 文件路径
5	+	./compare_manifests.py \
6	+	"/media/anduin/Ubuntu 25.04 amd64/casper/filesystem.manifest" \
7	+	"/media/anduin/Zorin OS 17.3 Core 64bit/casper/filesystem.manifest" \
8	+	"/media/anduin/anduinos/casper/filesystem.manifest"

compare_manifest.sh

			@@ -1,34 +1,151 @@
1		-	#!/usr/bin/env bash
2		-	# compare_manifest.sh - Compare two manifest files and list packages present in the first but not the second.
3		-
4		-	set -euo pipefail
5		-
6		-	# Function to display usage information
7		-	usage() {
8		-	echo "Usage: $0 <left.manifest> <right.manifest>"
9		-	echo
10		-	echo "Extracts package names (first column, without architecture suffix) from each manifest, sorts them,"
11		-	echo "and prints packages present in the left file but not the right."
12		-	exit 1
13		-	}
14		-
15		-	# Ensure exactly two arguments are provided
16		-	if [[ $# -ne 2 ]]; then
17		-	usage
18		-	fi
19		-
20		-	left_manifest="$1"
21		-	right_manifest="$2"
22		-
23		-	# Create a temporary directory for intermediate files
24		-	tmpdir=$(mktemp -d)
25		-	# Ensure the temporary directory is removed on script exit
26		-	trap 'rm -rf "${tmpdir}"' EXIT
27		-
28		-	# Extract package names, strip ':架构' 后缀，排序并去重，写入临时文件
29		-	awk '{print $1}' "${left_manifest}" \| sed 's/:.*$//' \| sort -u > "${tmpdir}/left.txt"
30		-	awk '{print $1}' "${right_manifest}" \| sed 's/:.*$//' \| sort -u > "${tmpdir}/right.txt"
31		-
32		-	# Output the comparison result
33		-	echo "Packages in '${left_manifest}' but not in '${right_manifest}':"
34		-	comm -23 "${tmpdir}/left.txt" "${tmpdir}/right.txt"
	1	+	#!/usr/bin/env python3
	2	+	# -- coding: utf-8 --
	3	+	"""
	4	+	compare_manifests.py
	5	+
	6	+	读取三个 filesystem.manifest 文件，比较包的并集，并输出包含 Section、Priority、Description 及各发行版存在情况的 Markdown 表格。
	7	+	增加详细日志输出，并使用多线程加速 apt show 调用。
	8	+	修复：自动剥离包名中的 ":架构" 后缀。
	9	+	"""
	10	+	import sys
	11	+	import subprocess
	12	+	import logging
	13	+	import concurrent.futures
	14	+	import time
	15	+
	16	+	# 日志配置
	17	+	logging.basicConfig(
	18	+	level=logging.INFO,
	19	+	format="%(asctime)s [%(levelname)s] %(message)s",
	20	+	datefmt="%Y-%m-%d %H:%M:%S"
	21	+	)
	22	+
	23	+
	24	+	def parse_manifest(path):
	25	+	"""
	26	+	读取 manifest 文件，返回其中所有包名的集合。
	27	+	自动剥离包名中的 ":<arch>" 后缀。
	28	+	"""
	29	+	pkgs = set()
	30	+	logging.info(f"Parsing manifest: {path}")
	31	+	try:
	32	+	with open(path, encoding='utf-8') as f:
	33	+	for line in f:
	34	+	line = line.strip()
	35	+	if not line or line.startswith('#'):
	36	+	continue
	37	+	parts = line.split()
	38	+	raw_pkg = parts[0]
	39	+	pkg = raw_pkg.split(':', 1)[0]
	40	+	pkgs.add(pkg)
	41	+	logging.info(f"Found {len(pkgs)} unique packages in {path}")
	42	+	except Exception as e:
	43	+	logging.error(f"Error reading manifest {path}: {e}")
	44	+	sys.exit(1)
	45	+	return pkgs
	46	+
	47	+
	48	+	def get_pkg_info(pkg):
	49	+	"""
	50	+	调用 `apt show pkg`，解析 Section、Priority、Description（首段），
	51	+	返回 dict 包含信息。
	52	+	若调用失败，返回空字段。
	53	+	"""
	54	+	logging.debug(f"Fetching apt info for package: {pkg}")
	55	+	try:
	56	+	p = subprocess.run(
	57	+	["apt", "show", pkg],
	58	+	stdout=subprocess.PIPE,
	59	+	stderr=subprocess.DEVNULL,
	60	+	text=True,
	61	+	encoding='utf-8',
	62	+	timeout=30
	63	+	)
	64	+	except subprocess.TimeoutExpired:
	65	+	logging.warning(f"Timeout fetching info for {pkg}")
	66	+	return {"Section": "", "Priority": "", "Description": ""}
	67	+
	68	+	if p.returncode != 0:
	69	+	logging.debug(f"apt show failed for {pkg}")
	70	+	return {"Section": "", "Priority": "", "Description": ""}
	71	+
	72	+	section = ""
	73	+	priority = ""
	74	+	desc_lines = []
	75	+	in_desc = False
	76	+
	77	+	for line in p.stdout.splitlines():
	78	+	if in_desc:
	79	+	if not line.strip():
	80	+	break
	81	+	desc_lines.append(line.strip())
	82	+	else:
	83	+	if line.startswith("Section:"):
	84	+	section = line.split(":", 1)[1].strip()
	85	+	elif line.startswith("Priority:"):
	86	+	priority = line.split(":", 1)[1].strip()
	87	+	elif line.startswith("Description:"):
	88	+	desc = line.split(":", 1)[1].strip()
	89	+	desc_lines.append(desc)
	90	+	in_desc = True
	91	+
	92	+	full_desc = " ".join(desc_lines)
	93	+	logging.debug(f"Obtained info for {pkg}: section={section}, priority={priority}")
	94	+	return {"Section": section, "Priority": priority, "Description": full_desc}
	95	+
	96	+
	97	+	def main():
	98	+	if len(sys.argv) != 4:
	99	+	print(f"Usage: {sys.argv[0]} <ubuntu_manifest> <zorin_manifest> <anduin_manifest>")
	100	+	sys.exit(1)
	101	+
	102	+	start_time = time.time()
	103	+	paths = {
	104	+	"Ubuntu": sys.argv[1],
	105	+	"Zorin": sys.argv[2],
	106	+	"Anduin": sys.argv[3],
	107	+	}
	108	+
	109	+	# 解析 manifests
	110	+	sets = {name: parse_manifest(path) for name, path in paths.items()}
	111	+	all_pkgs = sorted(set.union(*sets.values()))
	112	+	total = len(all_pkgs)
	113	+	logging.info(f"Total unique packages to process: {total}")
	114	+
	115	+	# 多线程获取 apt 信息
	116	+	pkg_infos = {}
	117	+	max_workers = min(10, total)
	118	+	logging.info(f"Starting ThreadPoolExecutor with {max_workers} workers")
	119	+
	120	+	with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
	121	+	future_to_pkg = {executor.submit(get_pkg_info, pkg): pkg for pkg in all_pkgs}
	122	+	for idx, future in enumerate(concurrent.futures.as_completed(future_to_pkg), 1):
	123	+	pkg = future_to_pkg[future]
	124	+	try:
	125	+	pkg_infos[pkg] = future.result()
	126	+	logging.info(f"[{idx}/{total}] Processed {pkg}")
	127	+	except Exception as e:
	128	+	logging.error(f"Error processing {pkg}: {e}")
	129	+	pkg_infos[pkg] = {"Section": "", "Priority": "", "Description": ""}
	130	+
	131	+	# 写入 Markdown
	132	+	out_path = "comp_result.md"
	133	+	logging.info(f"Writing results to {out_path}")
	134	+	with open(out_path, "w", encoding="utf-8") as out:
	135	+	out.write("\| Package \| Section \| Priority \| Description \| Ubuntu \| Zorin \| Anduin \|\n")
	136	+	out.write("\|---------\|---------\|----------\|-------------\|--------\|-------\|--------\|\n")
	137	+
	138	+	for pkg in all_pkgs:
	139	+	info = pkg_infos.get(pkg, {"Section": "", "Priority": "", "Description": ""})
	140	+	mark = lambda name: "√" if pkg in sets[name] else ""
	141	+	desc = info["Description"].replace("\|", "\\\|")
	142	+	out.write(
	143	+	f"\| {pkg} \| {info['Section']} \| {info['Priority']} \| {desc} \|"
	144	+	f" {mark('Ubuntu')} \| {mark('Zorin')} \| {mark('Anduin')} \|\n"
	145	+	)
	146	+
	147	+	elapsed = time.time() - start_time
	148	+	logging.info(f"Completed in {elapsed:.2f}s")
	149	+
	150	+	if __name__ == "__main__":
	151	+	main()

anduin revised this gist 6 months ago. Go to revision

1 file changed, 5 insertions, 5 deletions

compare_manifest.sh

			@@ -7,8 +7,8 @@ set -euo pipefail
7	7		usage() {
8	8		echo "Usage: $0 <left.manifest> <right.manifest>"
9	9		echo
10		-	echo "Extracts package names (first column) from each manifest, sorts them,"
11		-	echo "and prints packages present in the left file but not in the right."
	10	+	echo "Extracts package names (first column, without architecture suffix) from each manifest, sorts them,"
	11	+	echo "and prints packages present in the left file but not the right."
12	12		exit 1
13	13		}
14	14
			@@ -25,9 +25,9 @@ tmpdir=$(mktemp -d)
25	25		# Ensure the temporary directory is removed on script exit
26	26		trap 'rm -rf "${tmpdir}"' EXIT
27	27
28		-	# Extract package names (first column), sort, and write to temp files
29		-	awk '{print $1}' "${left_manifest}" \| sort > "${tmpdir}/left.txt"
30		-	awk '{print $1}' "${right_manifest}" \| sort > "${tmpdir}/right.txt"
	28	+	# Extract package names, strip ':架构' 后缀，排序并去重，写入临时文件
	29	+	awk '{print $1}' "${left_manifest}" \| sed 's/:.*$//' \| sort -u > "${tmpdir}/left.txt"
	30	+	awk '{print $1}' "${right_manifest}" \| sed 's/:.*$//' \| sort -u > "${tmpdir}/right.txt"
31	31
32	32		# Output the comparison result
33	33		echo "Packages in '${left_manifest}' but not in '${right_manifest}':"

anduin revised this gist 6 months ago. Go to revision

1 file changed, 34 insertions

compare_manifest.sh(file created)

		@@ -0,0 +1,34 @@
1	+	#!/usr/bin/env bash
2	+	# compare_manifest.sh - Compare two manifest files and list packages present in the first but not the second.
3	+
4	+	set -euo pipefail
5	+
6	+	# Function to display usage information
7	+	usage() {
8	+	echo "Usage: $0 <left.manifest> <right.manifest>"
9	+	echo
10	+	echo "Extracts package names (first column) from each manifest, sorts them,"
11	+	echo "and prints packages present in the left file but not in the right."
12	+	exit 1
13	+	}
14	+
15	+	# Ensure exactly two arguments are provided
16	+	if [[ $# -ne 2 ]]; then
17	+	usage
18	+	fi
19	+
20	+	left_manifest="$1"
21	+	right_manifest="$2"
22	+
23	+	# Create a temporary directory for intermediate files
24	+	tmpdir=$(mktemp -d)
25	+	# Ensure the temporary directory is removed on script exit
26	+	trap 'rm -rf "${tmpdir}"' EXIT
27	+
28	+	# Extract package names (first column), sort, and write to temp files
29	+	awk '{print $1}' "${left_manifest}" \| sort > "${tmpdir}/left.txt"
30	+	awk '{print $1}' "${right_manifest}" \| sort > "${tmpdir}/right.txt"
31	+
32	+	# Output the comparison result
33	+	echo "Packages in '${left_manifest}' but not in '${right_manifest}':"
34	+	comm -23 "${tmpdir}/left.txt" "${tmpdir}/right.txt"

Newer Older